From 3cee395037300ec454f1b87d841ce631268efd88 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassiedoll@users.noreply.github.com>
Date: Tue, 27 May 2014 16:03:08 -0700
Subject: [PATCH 01/42] Update README.rst

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 0148fb7..e5ff6c2 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,7 @@ This Python client provides a simple web application which fetches data from the
 interface.
 
 It can be run with app engine or without.
-See `the docs <google-genomics.readthedocs.org/en/latest/api-client-python>`_
+See `the docs <http://google-genomics.readthedocs.org/en/latest/api-client-python>`_
 for more information.
 
 .. _Google Genomics Api: https://developers.google.com/genomics

From 4c5dc0d0eef353f209afb15e1dfb5461879b39a4 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Mon, 9 Jun 2014 09:40:50 -0700
Subject: [PATCH 02/42] First attempt at incorporating variant data

- generalizes ui to show and search over readsets and callsets
- supports multiple callsets at a time (still limited to one readset
for now)
- height of the graph now adjusts to accommodate the available data
- lots of cleanup still needed
- big todo for supporting data from multiple backends (will prob need
to change the hash format a bit)
---
 main.html              |  61 ++++++---
 main.py                |  60 ++++++---
 static/css/main.css    |  24 +++-
 static/js/main.js      | 232 +++++++++++++++++++++++++--------
 static/js/readgraph.js | 282 ++++++++++++++++++++++++++++++-----------
 5 files changed, 484 insertions(+), 175 deletions(-)

diff --git a/main.html b/main.html
index 0564556..1022ae8 100644
--- a/main.html
+++ b/main.html
@@ -53,11 +53,11 @@ <h4 class="modal-title">About GABrowse</h4>
         <p class="lead">GABrowse is a sample application designed to demonstrate the capabilities of the
         <a href="http://ga4gh.github.io/apis/reads/v0.1/">GA4GH Reads API v0.1</a>.</p>
 
-        <p>Currently, you can view Reads from Google, NCBI and EBI.</p>
+        <p>Currently, you can view data from Google, NCBI and EBI.</p>
 
         <p>
           <ul>
-            <li>Use the button on the left to select a Readset.</li>
+            <li>Use the button on the left to select a Readset or Callset.</li>
             <li>Once loaded, choose a chromosome and zoom or drag the main graph to
               explore <a href="http://en.wikipedia.org/wiki/DNA_sequence">Read data.</a></li>
             <li>Individual <a href="http://en.wikipedia.org/wiki/Nucleobase">bases</a>
@@ -76,12 +76,12 @@ <h4 class="modal-title">About GABrowse</h4>
   </div>
 </div>
 
-<div class="modal" id="readsetSearch">
+<div class="modal" id="setSearch">
   <div class="modal-dialog">
     <div class="modal-content">
       <div class="modal-header">
         <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
-        <h4 class="modal-title">Select a readset to view</h4>
+        <h4 class="modal-title">Select data to view</h4>
       </div>
       <div class="modal-body">
         <form class="form-inline" style="margin-bottom: 10px">
@@ -97,7 +97,7 @@ <h4 class="modal-title">Select a readset to view</h4>
           <div class="form-group">
             {% for backend, data in backends.iteritems() %}
               <!-- TODO: Simplify this -->
-              <select class="form-control datasetSelector" id="datasetId{{ backend }}">
+              <select class="form-control datasetSelector" id="datasetId{{ backend }}" supportsCallsets="{{ data.supportsCallsets }}">
               {% for dataset, datasetId in data.datasets.iteritems() %}
                   <option value="{{ datasetId }}"> {{ dataset }} </option>
                 {% endfor %}
@@ -105,14 +105,30 @@ <h4 class="modal-title">Select a readset to view</h4>
             {% endfor %}
           </div>
           <div class="form-group">
-            <input type="text" class="form-control" placeholder="name" id="readsetName"/>
+            <input type="text" class="form-control" placeholder="name" id="setName"/>
           </div>
-          <button type="submit" class="btn btn-primary" onclick="return searchReadsets(this)">Search</button>
+          <button type="submit" class="btn btn-primary" onclick="return searchSets(this)">Search</button>
         </form>
-        <div id="readsetResults" class="list-group">
-          <img src="static/img/spinner.gif"/>
+        <div>
+          <ul class="nav nav-tabs">
+            <li class="active" id="READSETTab"><a>Readsets</a></li>
+            <li class="disabled" id="CALLSETTab"><a>Callsets</a></li>
+          </ul>
+          <div class="tab-content">
+            <div class="tab-pane active" id="searchPaneREADSET">
+              <div class="list-group results">
+                <img src="static/img/spinner.gif"/>
+              </div>
+              <div class="paginationContainer"></div>
+            </div>
+            <div class="tab-pane" id="searchPaneCALLSET">
+              <div class="list-group results">
+                <img src="static/img/spinner.gif"/>
+              </div>
+              <div class="paginationContainer"></div>
+            </div>
+          </div>
         </div>
-        <div id="readsetPagination"></div>
       </div>
       <div class="modal-footer">
         <button type="button" class="btn btn-default" data-dismiss="modal">Done</button>
@@ -124,32 +140,35 @@ <h4 class="modal-title">Select a readset to view</h4>
 <!-- Read browser -->
 <div class="toggleable reads">
   <div class="col-md-2">
-    <h4>Readsets</h4>
-    <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#readsetSearch">
-      <span class="glyphicon glyphicon-plus"/> Add
+    <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#setSearch">
+      <span class="glyphicon glyphicon-plus"></span> Add data
     </button>
-    <ul class="list-group" id="activeReadsets" style="margin-top: 20px">
 
-    </ul>
+    <h5 id="CALLSETTitle" style="display:none">Callsets</h5>
+    <ul class="list-group" id="activeCALLSET" style="margin-top: 20px"></ul>
+
+    <h5 id="READSETTitle" style="display:none">Readsets</h5>
+    <ul class="list-group" id="activeREADSET" style="margin-top: 20px"></ul>
+
+
     <div class="well" style="display: none" id="jumpDiv">
       <form role="form">
         <div class="form-group">
           <input type="text" class="form-control" name="name" placeholder="TAS2R38 or rs1726866 or 150,000" id="readsetPosition">
         </div>
         <button type="submit" class="btn btn-primary"
-                onclick="switchToLocation($('#readsetPosition').val()); return false;">Jump to position</button>
+                onclick="updateUserLocation($('#readsetPosition').val()); return false;">Jump to position</button>
         <div id="jumpResults" class="list-group"></div>
       </form>
     </div>
 
-    <div class="well" style="display: none" id="readDiv">
-
-    </div>
+    <div class="well infoDiv" style="display: none" id="variantDiv"></div>
+    <div class="well infoDiv" style="display: none" id="readDiv"></div>
   </div>
 
   <div class="col-md-10">
-    <div id="chooseReadsetMessage">
-      <h3><small>Add a readset to get started.</small></h3>
+    <div id="chooseSetMessage">
+      <h3><small>Add a readset or callset to get started.</small></h3>
     </div>
     <div id="sequences"></div>
     <svg id="graph"></svg>
diff --git a/main.py b/main.py
index 2e15e6d..d178722 100644
--- a/main.py
+++ b/main.py
@@ -57,8 +57,9 @@
   SUPPORTED_BACKENDS['GOOGLE'] = {
     'name': 'Google',
     'url': 'https://www.googleapis.com/genomics/v1beta/%s?key='
-           + GOOGLE_API_KEY + "&%s",
+           + GOOGLE_API_KEY + '&%s',
     'supportsNameFilter': True,
+    #'supportsCallsets': True,
     'datasets': {'1000 Genomes': '376902546192',
                  'DREAM SMC Challenge': '337315832689',
                  'PGP': '383928317087',
@@ -96,7 +97,7 @@ def supports_name_filter(self):
   def get_base_api_url(self):
     return SUPPORTED_BACKENDS[self.get_backend()]['url']
 
-  def get_content(self, path, method='POST', body=None, params=""):
+  def get_content(self, path, method='POST', body=None, params=''):
     response, content = http.request(
       uri= self.get_base_api_url() % (path, params),
       method=method, body=json.dumps(body) if body else None,
@@ -105,12 +106,12 @@ def get_content(self, path, method='POST', body=None, params=""):
     try:
       content = json.loads(content)
     except ValueError:
-      logging.error("non-json api content %s" % content[:1000])
+      logging.error('non-json api content %s' % content[:1000])
       raise ApiException('The API returned invalid JSON')
 
     if response.status >= 300:
-      logging.error("error api response %s" % response)
-      logging.error("error api content %s" % content)
+      logging.error('error api response %s' % response)
+      logging.error('error api content %s' % content)
       if 'error' in content:
         raise ApiException(content['error']['message'])
       else:
@@ -121,11 +122,14 @@ def write_content(self, path, method='POST', body=None):
     self.response.write(json.dumps(self.get_content(path, method, body)))
 
 
-class ReadsetSearchHandler(BaseRequestHandler):
+class SetSearchHandler(BaseRequestHandler):
 
   def get(self):
-    readset_id = self.request.get('readsetId')
-    if not readset_id:
+    use_callsets = self.request.get('setType') == 'CALLSET'
+    set_type = 'callsets' if use_callsets  else 'readsets'
+    set_id = self.request.get('setId')
+
+    if not set_id:
       dataset_id = self.request.get('datasetId')
       name = self.request.get('name')
       if dataset_id:
@@ -137,25 +141,33 @@ def get(self):
       if self.supports_name_filter():
         body['name'] = name
 
-      response = self.get_content("readsets/search", body=body,
-                                  params="fields=readsets(id,name)")
+      response = self.get_content('%s/search' % set_type, body=body,
+                                  params='fields=%s(id,name)' % set_type)
 
       if not self.supports_name_filter() and name:
         name = name.lower()
-        response['readsets'] = [r for r in response['readsets']
+        response[set_type] = [r for r in response[set_type]
                                 if name in r['name'].lower()]
       self.response.write(json.dumps(response))
       return
 
-    # Single readset response
-    self.write_content("readsets/%s" % readset_id, method='GET')
+    # Single object response
+    if use_callsets:
+      # TODO: Use GET /callsets/id and GET /variants/summary
+      response = {'id' : set_id, 'name' : set_id, 'contigs': [
+        {'name' : '22', 'length': 51304566}
+      ]}
+      self.response.write(json.dumps(response))
+      return
+
+    self.write_content('%s/%s' % (set_type, set_id), method='GET')
 
 
 class ReadSearchHandler(BaseRequestHandler):
 
   def get(self):
     body = {
-      'readsetIds': self.request.get('readsetIds').split(','),
+      'readsetIds': self.request.get('setIds').split(','),
       'sequenceName': self.request.get('sequenceName'),
       'sequenceStart': max(0, int(self.request.get('sequenceStart'))),
       'sequenceEnd': int(self.request.get('sequenceEnd')),
@@ -163,7 +175,22 @@ def get(self):
     pageToken = self.request.get('pageToken')
     if pageToken:
       body['pageToken'] = pageToken
-    self.write_content("reads/search", body=body)
+    self.write_content('reads/search', body=body)
+
+
+class VariantSearchHandler(BaseRequestHandler):
+
+  def get(self):
+    body = {
+      'callsetIds': self.request.get('setIds').split(','),
+      'contig': self.request.get('sequenceName'),
+      'startPosition': max(0, int(self.request.get('sequenceStart'))),
+      'endPosition': int(self.request.get('sequenceEnd')),
+    }
+    pageToken = self.request.get('pageToken')
+    if pageToken:
+      body['pageToken'] = pageToken
+    self.write_content('variants/search', body=body)
 
 
 class BaseSnpediaHandler(webapp2.RequestHandler):
@@ -258,7 +285,8 @@ def get(self):
     [
      ('/', MainHandler),
      ('/api/reads', ReadSearchHandler),
-     ('/api/readsets', ReadsetSearchHandler),
+     ('/api/variants', VariantSearchHandler),
+     ('/api/sets', SetSearchHandler),
      ('/api/snps', SnpSearchHandler),
      ('/api/alleles', AlleleSearchHandler),
     ],
diff --git a/static/css/main.css b/static/css/main.css
index be2e659..37996d6 100644
--- a/static/css/main.css
+++ b/static/css/main.css
@@ -50,6 +50,7 @@ limitations under the License.
 #sequences .sequence .title {
   white-space: normal;
   word-wrap: break-word;
+  display: inline-block;
 }
 
 #sequences .sequence .summary {
@@ -60,19 +61,32 @@ limitations under the License.
   color: white;
 }
 
-#readDiv h4 {
+#sequences .sequence .badge {
+  background-color: #ddd;
+}
+
+#sequences .sequence.active .badge {
+  background-color: white;
+  color: #428bca;
+}
+
+.infoDiv h4 {
   word-wrap: break-word;
 }
 
-#readsetSearch .modal-body {
+#setSearch .modal-body {
   padding-bottom: 0;
 }
 
-#readsetPagination {
+#setSearch .results {
+  padding-top: .75em;
+}
+
+.paginationContainer {
   text-align: center;
 }
 
-#readsetPagination ul {
+.paginationContainer ul {
   margin: 0;
 }
 
@@ -96,7 +110,7 @@ limitations under the License.
 svg {
   margin-top: 20px;
   width: 100%;
-  height: 800px;
+  height: 800px; /* Height is updated in readgraph.js */
   display: none;
 }
 .axis line,
diff --git a/static/js/main.js b/static/js/main.js
index bcda8ae..773204a 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -14,6 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+var CALLSET_TYPE = "CALLSET";
+var READSET_TYPE = "READSET";
+
 function toggleUi(clazz, link) {
   $(".toggleable").hide();
   $("." + clazz).show();
@@ -46,72 +49,140 @@ function closeButton() {
   return $('<button type="button" class="close" aria-hidden="true">&times;</button>');
 }
 
-function addReadset(backend, id, opt_location) {
-  if (readgraph.hasReadset(id)) {
-    if (opt_location) {
-      readgraph.jumpGraph(opt_location);
+var loadedSetData = {};
+function loadSet(backend, readsetIds, callsetIds, opt_location, setType, id) {
+  if (_.has(loadedSetData, id)) {
+    return false;
+  }
+
+  showMessage('Loading data');
+  
+  $.getJSON('/api/sets', {backend: backend, setType: setType, setId: id})
+    .done(function(res) {
+      var sequenceData = res.contigs || res.fileData[0].refSequences;
+      loadedSetData[id] = {id: id, name: res.name, type: setType,
+        backend: backend, sequences: sequenceData};
+      updateSets(backend, readsetIds, callsetIds, opt_location);
+    });
+  return true;
+}
+
+function updateSets(backend, readsetIds, callsetIds, opt_location) {
+  // Load missing readsets
+  for (var i = 0; i < readsetIds.length; i++) {
+    if (loadSet(backend, readsetIds, callsetIds, opt_location, READSET_TYPE, readsetIds[i])) {
+      // Wait for the set to callback
+      return;
     }
-    return;
   }
-  showMessage('Loading readset');
 
-  $.getJSON('/api/readsets', {'backend' : backend, 'readsetId': id})
-      .done(function(res) {
-        readgraph.addReadset(backend, id, res.fileData[0].refSequences);
-        if (opt_location) {
-          readgraph.jumpGraph(opt_location);
-        }
+  // Load missing callsets
+  for (var j = 0; j < callsetIds.length; j++) {
+    if (loadSet(backend, readsetIds, callsetIds, opt_location, CALLSET_TYPE, callsetIds[j])) {
+      // Wait for the set to callback
+      return;
+    }
+  }
 
-        // TODO: Get this data from a better location
-        var backendName = $('#backend option[value=' + backend + ']').text().trim();
-        var name = backendName + ": " + res.name;
-
-        var readsetList = $('#activeReadsets').empty();
-        var li = $('<li>', {'id': 'readset-' + id, 'class': 'list-group-item'})
-            .text(name).appendTo(readsetList);
-        closeButton().appendTo(li).click(function() {
-          li.remove();
-          readgraph.removeReadset(id);
-          return false;
-        });
-      });
+  // TODO: Get this data from a better location
+  // TODO: Allow data from different backends to be loaded at the same time
+  var backendName = $('#backend option[value=' + backend + ']').text().trim();
+
+  updateListItems(backendName, READSET_TYPE, readsetIds, loadedSetData);
+  updateListItems(backendName, CALLSET_TYPE, callsetIds, loadedSetData);
+
+  // Update readgraph with new sets
+  var setData = _.filter(loadedSetData, function(data) {
+    return (_.contains(readsetIds, data.id) && data.type == READSET_TYPE) ||
+      (_.contains(callsetIds, data.id) && data.type == CALLSET_TYPE);
+  });
+
+  readgraph.updateSets(setData);
+  if (setData.length > 0 && opt_location) {
+    readgraph.jumpGraph(opt_location);
+  }
+}
+
+function updateListItems(backendName, setType, ids, loadedSetData) {
+  $('#' + setType + 'Title').toggle(ids.length > 0);
+  var setList = $('#active' + setType).empty();
+
+  $.each(ids, function(i, id) {
+    var name = loadedSetData[id].name;
+
+    var li = $('<li>', {'id': setType + '-' + id, 'class': 'list-group-item'})
+      .appendTo(setList);
+
+    closeButton().appendTo(li).click(function() {
+      removeSet(id, setType);
+      return false;
+    });
+
+    var displayName = backendName + ": " + name;
+    $('<div/>', {'class': 'setName'}).text(displayName).appendTo(li);
+  });
 }
 
-function searchReadsets(button) {
+function searchSets(button) {
   if (button) {
     button = $(button);
     button.button('loading');
   }
 
-  var div = $('#readsetResults').html('<img src="static/img/spinner.gif"/>');
   var backend = $('#backend').val();
-  var datasetId = $('#datasetId' + backend).val();
+  var datasetSelector = $('#datasetId' + backend);
+  var datasetId = datasetSelector.val();
+  var supportsCallsets = datasetSelector.attr("supportsCallsets");
+
+  searchSetsOfType(button, READSET_TYPE, backend, datasetId);
+
+  $("#CALLSETTab").toggleClass("disabled", !supportsCallsets);
+  if (supportsCallsets) {
+    searchSetsOfType(button, CALLSET_TYPE, backend, datasetId);
+  } else {
+    // Make sure the readset tab is selected
+    $("#READSETTab").click();
+  }
+}
+
+function setSearchTab(setType) {
+  $('.tab-pane').hide();
+  $('.nav-tabs li').removeClass("active");
+  $('#' + setType + 'Tab').addClass('active');
+  $('#searchPane' + setType).show();
+}
+
+function searchSetsOfType(button, setType, backend, datasetId) {
+  var tabPane = $('#searchPane' + setType);
+  var div = tabPane.find('.results')
+    .html('<img src="static/img/spinner.gif"/>');
 
   function getItemsOnPage(page) {
-    return $('#readsetResults .list-group-item[page=' + page + ']');
+    return div.find('.list-group-item[page=' + page + ']');
   }
 
-  var readsetsPerPage = 10;
-  $.getJSON('/api/readsets', {'backend': backend, 'datasetId': datasetId,
-      'name': $('#readsetName').val()})
+  var setsPerPage = 10;
+  $.getJSON('/api/sets', {'backend': backend, 'datasetId': datasetId,
+      'setType': setType, 'name': $('#setName').val()})
       .done(function(res) {
         div.empty();
 
-        var pagination = $('#readsetPagination');
+        var pagination = tabPane.find('.paginationContainer');
         pagination.hide();
 
-        if (!res.readsets) {
-          div.html('No readsets found');
+        var sets = res.readsets || res.callsets;
+        if (!sets) {
+          div.html('No data found');
           return;
         }
 
-        var totalPages = Math.ceil(res.readsets.length / readsetsPerPage);
+        var totalPages = Math.ceil(sets.length / setsPerPage);
 
-        $.each(res.readsets, function(i, data) {
-          var page = Math.floor(i / readsetsPerPage) + 1;
+        $.each(sets, function(i, data) {
+          var page = Math.floor(i / setsPerPage) + 1;
           $('<a/>', {'href': '#', 'class': 'list-group-item', 'page': page})
               .text(data.name).appendTo(div).click(function() {
-            switchToReadset(backend, data.id);
+            switchToSet(backend, setType, data.id);
             return false;
           }).hide();
         });
@@ -124,7 +195,7 @@ function searchReadsets(button) {
             total: totalPages,
             maxVisible: 10
           }).on("page", function(event, newPage) {
-            $('#readsetResults .list-group-item').hide();
+            div.find('.list-group-item').hide();
             getItemsOnPage(newPage).show();
           });
         }
@@ -138,26 +209,55 @@ function searchReadsets(button) {
 
 // Hash functions
 function setAnchor(map) {
-  window.location.hash = $.param(map);
+  window.location.hash = $.param(map, true);
 }
 
+var arrayKeys = ['readsetId', 'callsetId'];
 function getAnchorMap() {
   var hashParts = window.location.hash.substring(1).split('&');
   var map = {};
   for (var i = 0; i < hashParts.length; i++) {
     var option = decodeURIComponent(hashParts[i]).split('=');
-    map[option[0]] = option[1]
+    var key = option[0];
+    var value = option[1];
+
+    if (!_.contains(arrayKeys, key)) {
+      map[key] = value;
+    } else if (map[key]) {
+      map[key].push(value);
+    } else {
+      map[key] = [value];
+    }
   }
 
   return map;
 }
 
-function switchToReadset(backend, id) {
-  setAnchor({
-    backend: backend,
-    readsetId: id
-  });
-  $('#readsetSearch').modal('hide');
+function removeSet(id, setType) {
+  var state = getAnchorMap();
+  var key = setType == READSET_TYPE ? 'readsetId' : 'callsetId';
+  state[key] = _.without(state[key], id);
+  if (state[key].length == 0) {
+    delete state[key];
+  }
+  setAnchor(state);
+}
+
+function switchToSet(backend, setType, id) {
+  var state = getAnchorMap();
+  var key = setType == READSET_TYPE ? 'readsetId' : 'callsetId';
+
+  if (setType == READSET_TYPE) {
+    state[key] = [id];
+  } else {
+    state[key] = (state[key] || []);
+    state[key].push(id);
+  }
+
+  // TODO: Support multiple backends at once
+  state.backend = backend;
+  setAnchor(state);
+  $('#setSearch').modal('hide');
 }
 
 function switchToLocation(location) {
@@ -165,23 +265,29 @@ function switchToLocation(location) {
   setAnchor(state);
 }
 
+function updateUserLocation(location) {
+  switchToLocation(readgraph.getCurrentSequence() + ":" + location);
+}
+
 function handleHash() {
   var state = getAnchorMap();
   if (state.backend) {
     $("#backend").val(state.backend);
-    if (state.readsetId) {
-      addReadset(state.backend, state.readsetId, state.location);
 
-      if (state.location) {
-        $("#readsetPosition").val(state.location);
-      }
+    updateSets(state.backend, (state.readsetId || []).slice(0, 1),
+      state.callsetId || [], state.location);
+    if (state.location) {
+      // Strip off the chromosome prefix
+      var colonIndex = state.location.indexOf(":");
+      var location = state.location.substring(colonIndex + 1);
+      $("#readsetPosition").val(location);
     }
   }
 }
 
 
 // Show the about popup when the page loads, read the hash,
-// and prep the initial readset search
+// and prep the initial set search
 $(document).ready(function() {
   $('#about').modal('show');
 
@@ -195,7 +301,21 @@ $(document).ready(function() {
 
   $('#backend').change(function() {
     $('.datasetSelector').hide();
-    $('#datasetId' + $(this).val()).show()
+    $('#datasetId' + $(this).val()).show();
   }).change();
-  searchReadsets();
+
+  // TODO: Simplify with general searchTab classes
+  $("#READSETTab").click(function() {
+    setSearchTab(READSET_TYPE);
+    return false;
+  });
+  $("#CALLSETTab").click(function() {
+    if ($(this).hasClass("disabled")) {
+      return false;
+    }
+    setSearchTab(CALLSET_TYPE);
+    return false;
+  });
+
+  searchSets();
 });
\ No newline at end of file
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index b8c80c8..b0c68db 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -19,7 +19,7 @@ var readgraph = new function() {
 
   var width = 0;
   var height = 800;
-  var margin = 30;
+  var margin = 40;
 
   var textHeight, textWidth = 0;
 
@@ -37,15 +37,38 @@ var readgraph = new function() {
   var unsupportedMessage = null;
 
   // Current state
-  var readsetIds = [];
-  var readsetBackend = null;
-  var sequences = null;
+  // Format: {id, type, backend, sequences}
+  var setObjects = [];
+  var mergedSequences = [];
   var currentSequence = null;
   var readStats = {}; // Map from position to stat information
   var xhrTimeout = null;
 
+  var readTrackLength = 0;
+  var callsetTrackLength = 0;
+
   // Dom elements
-  var svg, positionIndicator, readGroup, readDiv, spinner = null;
+  var svg, axisGroup, readGroup, readDiv, variantDiv, spinner = null;
+  var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
+
+  var updateHeight = function() {
+    var totalTracks = readTrackLength + callsetTrackLength;
+    height = totalTracks * textHeight + 100;
+    height = Math.max(height, 450);
+
+    y.range([margin, height - margin*2]).domain([totalTracks, -1]);
+    $('#graph').height(height);
+
+    // TODO: Reduce duplicate height setting code
+    axisGroup.attr('transform', 'translate(0,' + (height - margin) + ')');
+    positionIndicatorBg.attr('height', height - margin);
+    positionIndicatorText.attr('y', height - margin - textHeight);
+    hoverline.attr("y2", height);
+
+    zoom.size([width, height]);
+
+    return totalTracks;
+  };
 
   var getScaleLevel = function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
@@ -91,11 +114,12 @@ var readgraph = new function() {
     minRange = (width / textWidth / 2); // Twice the zoom of individual bases
 
     readDiv = $('#readDiv');
+    variantDiv = $('#variantDiv');
 
 
     // Svg init
     // Reads Axis
-    svg.append('g')
+    axisGroup = svg.append('g')
         .attr('transform', 'translate(0,' + (height - margin) + ')')
         .attr('class', 'axis');
 
@@ -104,7 +128,7 @@ var readgraph = new function() {
         width/2, height/4);
 
     // Hover line
-    var hoverline = svg.append("line")
+    hoverline = svg.append("line")
         .attr("class", "hover hoverline")
         .attr("x1", 0).attr("x2", 0)
         .attr("y1", 0).attr("y2", height);
@@ -144,11 +168,11 @@ var readgraph = new function() {
     positionIndicator = svg.append('g')
         .attr('transform', 'translate(0,0)')
         .attr('class', 'axis');
-    positionIndicator.append('rect')
+    positionIndicatorBg = positionIndicator.append('rect')
         .attr('class', 'positionIndicator background')
         .attr('x', 0).attr('y', 0)
         .attr('width', textWidth * 1.5).attr('height', height - margin);
-    positionIndicator.append('text')
+    positionIndicatorText = positionIndicator.append('text')
         .attr('class', 'positionIndicator text')
         .attr('x', 3)
         .attr('y', height - margin - textHeight);
@@ -260,12 +284,12 @@ var readgraph = new function() {
   };
 
   var fuzzyFindSequence = function(chr) {
-    var actualNames = _.pluck(sequences, 'name');
+    var actualNames = _.pluck(mergedSequences, 'name');
     var possibleNames = [chr, "chr" + chr];
     possibleNames = _.intersection(actualNames, possibleNames);
 
     if (possibleNames.length > 0) {
-      return _.findWhere(sequences, {name: possibleNames[0]});
+      return _.findWhere(mergedSequences, {name: possibleNames[0]});
     }
     return null;
   };
@@ -275,7 +299,7 @@ var readgraph = new function() {
       // Update our sequence
       var sequence = fuzzyFindSequence(chr);
       if (!sequence) {
-        showError('This readset doesn\'t have the sequence ' + chr +
+        showError('This data doesn\'t have the sequence ' + chr +
           '. Please try a different position.');
         return;
       }
@@ -351,10 +375,17 @@ var readgraph = new function() {
     return '/static/img/' + name + '.png';
   };
 
+  var getSequenceName = function(sequence) {
+    return sequence.name;
+  };
+
   var updateSequences = function() {
     var sequencesDiv = $("#sequences").empty();
+    var allSequences = _.flatten(_.pluck(setObjects, 'sequences'));
+    var indexedSequences = _.countBy(allSequences, getSequenceName);
+    mergedSequences = _.uniq(allSequences, false, getSequenceName);
 
-    $.each(sequences, function(i, sequence) {
+    $.each(mergedSequences, function(i, sequence) {
       var title, imageUrl;
 
       if (sequence.name.indexOf('X') != -1) {
@@ -374,6 +405,7 @@ var readgraph = new function() {
       }
 
       var summary = xFormat(sequence['length']) + " bases";
+      var setCount = indexedSequences[sequence.name];
 
       var sequenceDiv = $('<div/>', {'class': 'sequence',
         id: sequenceId(sequence.name)}).appendTo(sequencesDiv);
@@ -381,6 +413,10 @@ var readgraph = new function() {
         $('<img>', {'class': 'pull-left', src: imageUrl}).appendTo(sequenceDiv);
       }
       $('<div>', {'class': 'title'}).text(title).appendTo(sequenceDiv);
+      if (setObjects.length != setCount) {
+        $('<div>', {'class': 'badge pull-right'}).text(setCount)
+          .appendTo(sequenceDiv);
+      }
       $('<div>', {'class': 'summary'}).text(summary).appendTo(sequenceDiv);
 
       sequenceDiv.click(function() {
@@ -391,7 +427,9 @@ var readgraph = new function() {
     $('#jumpDiv').show();
   };
 
-  var updateDisplay = function(opt_skipReadQuery) {
+  var updateDisplay = function(opt_skipDataQuery) {
+    var maxY = updateHeight();
+
     var scaleLevel = getScaleLevel();
     var summaryView = scaleLevel < 2;
     var coverageView = scaleLevel == 2 || scaleLevel == 3;
@@ -399,6 +437,7 @@ var readgraph = new function() {
     var baseView = scaleLevel > 5;
 
     var reads = readGroup.selectAll(".read");
+    var variants = readGroup.selectAll(".variant");
     var outlines = reads.selectAll(".outline");
     var letters = reads.selectAll(".letter");
 
@@ -406,12 +445,13 @@ var readgraph = new function() {
     toggleVisibility(outlines, readView);
     toggleVisibility(letters, baseView);
     toggleVisibility(positionIndicator, baseView);
+    toggleVisibility(variants, baseView);
 
     var sequenceStart = parseInt(x.domain()[0]);
     var sequenceEnd = parseInt(x.domain()[1]);
 
-    if (!opt_skipReadQuery && (readView || baseView)) {
-      queryReads(sequenceStart, sequenceEnd);
+    if (!opt_skipDataQuery && (readView || baseView)) {
+      queryData(sequenceStart, sequenceEnd);
     }
 
     // TODO: Bring back coverage and summary views
@@ -419,6 +459,7 @@ var readgraph = new function() {
       outlines.attr("points", outlinePoints);
 
     } else if (baseView) {
+      // Read bases
       letters.style('display', function(data, i) {
             if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
               return 'none';
@@ -432,10 +473,13 @@ var readgraph = new function() {
           .attr("y", function(data, i) {
             return y(data.ry) + textHeight/2;
           });
+
+      // Red position highlight box
       var position = positionIndicator.attr('position');
       var indicatorX = x(position) + textWidth/2 - 2;
       positionIndicator.attr('transform', 'translate(' + indicatorX + ',0)');
 
+      // Read base stats
       var snp = positionIndicator.attr('snp');
       var loaded = positionIndicator.attr('loaded');
       if (!loaded && snp && readStats[position]) {
@@ -450,6 +494,21 @@ var readgraph = new function() {
           }
         });
       }
+
+      // Variants
+      variants.style('display', function(data, i) {
+            if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
+              return 'none';
+            } else {
+              return 'block';
+            }
+          })
+          .attr("x", function(data, i) {
+            return x(data.rx);
+          })
+          .attr("y", function(data, i) {
+            return y(maxY - data.ry) + textHeight/2;
+          });
     }
   };
 
@@ -520,30 +579,45 @@ var readgraph = new function() {
     return stringifyPoints(points);
   };
 
-  // Read details
+  // Hover details
+  var addField = function(dl, title, field) {
+    if (field) {
+      $("<dt/>").text(title).appendTo(dl);
+      $("<dd/>").text(field).appendTo(dl);
+    }
+  };
+
   var showRead = function(read, i) {
     readDiv.empty().show();
 
     $("<h4/>").text("Read: " + read.name).appendTo(readDiv);
     var dl = $("<dl/>").addClass("dl").appendTo(readDiv);
 
-    var addField = function(title, field) {
-      if (field) {
-        $("<dt/>").text(title).appendTo(dl);
-        $("<dd/>").text(field).appendTo(dl);
-      }
-    };
+    addField(dl, "Position", read.position);
+    addField(dl, "Length", read.length);
+    addField(dl, "Mate position", read.matePosition);
+    addField(dl, "Mapping quality", read.mappingQuality);
+    addField(dl, "Cigar", read.cigar);
+
+    d3.select(this).classed("selected", true);
+  };
+
+  var showVariant = function(data) {
+    variantDiv.empty().show();
+    var variant = data.variant;
+    var call = variant.calls[data.callIndex];
 
-    addField("Position", read.position);
-    addField("Length", read.length);
-    addField("Mate position", read.matePosition);
-    addField("Mapping quality", read.mappingQuality);
-    addField("Cigar", read.cigar);
+    $("<h4/>").text("Variant: " + variant.names.join(", "))
+      .appendTo(variantDiv);
+    var dl = $("<dl/>").addClass("dl").appendTo(variantDiv);
+
+    addField(dl, "Callset name", call.callsetName);
+    addField(dl, "Position", variant.position);
 
     d3.select(this).classed("selected", true);
   };
 
-  var hideRead = function(read, i) {
+  var deselectObject = function(read, i) {
     d3.select(this).classed("selected", false);
   };
 
@@ -555,17 +629,57 @@ var readgraph = new function() {
     }
   };
 
-  var clearReads = function() {
-    if (readGroup) {
-      readGroup.selectAll('.read').remove();
-    }
-    if (readDiv) {
-      readDiv.hide();
-    }
-    if (positionIndicator) {
-      toggleVisibility(positionIndicator, false);
+  var getGenotype = function(variant, call) {
+
+    // TODO: Switch to genotype field when its ready
+    var genotype = [];
+    var splits = call.info["GT"][0].split(/[|\/]/);
+    for (var g = 0; g < splits.length; g++) {
+      var allele = splits[g];
+      if (allele == 0) {
+        genotype.push(variant.referenceBases);
+      } else {
+        genotype.push(variant.alternateBases[allele - 1]);
+      }
     }
-    readStats = {};
+
+    return genotype;
+  };
+
+  var setVariants = function(variants) {
+    var data = [];
+    var maxCalls = 0;
+
+    $.each(variants, function(i, variant) {
+      maxCalls = Math.max(variant.calls.length, maxCalls);
+
+      $.each(variant.calls, function(callIndex, call) {
+        data.push({
+          id: variant.id + call.callsetId,
+          rx: variant.position,
+          ry: callIndex,
+          genotype: getGenotype(variant, call).join(";"),
+          variant: variant,
+          callIndex: callIndex
+          // TODO: Use likelihood for opacity
+        });
+      });
+
+    });
+
+    var variantDivs = readGroup.selectAll(".variant").data(data,
+        function(data){ return data.id; });
+
+    variantDivs.enter().append("text")
+        .attr('class', 'variant')
+        .text(function(data, i) { return data.genotype; })
+        .on("mouseover", showVariant)
+        .on("mouseout", deselectObject);
+
+    variantDivs.exit().remove();
+
+    callsetTrackLength = maxCalls;
+    updateDisplay(true);
   };
 
   var setReads = function(reads) {
@@ -661,7 +775,7 @@ var readgraph = new function() {
       yTracks.push(read.end);
     });
 
-    y.domain([yTracks.length, -1]);
+    readTrackLength = yTracks.length;
 
     readGroup.selectAll('.read').remove();
     if (reads.length == 0) {
@@ -676,7 +790,7 @@ var readgraph = new function() {
         .attr('class', 'read')
         .attr('index', function(read, i) { return read.index; })
         .on("mouseover", showRead)
-        .on("mouseout", hideRead);
+        .on("mouseout", deselectObject);
 
     var outlines = reads.selectAll('.outline')
         .data(function(read, i) { return [read];});
@@ -698,44 +812,60 @@ var readgraph = new function() {
   };
 
   var makeQueryParams = function(sequenceStart, sequenceEnd, type) {
+    var sets = _.where(setObjects, {type: type});
+    if (sets.length == 0) {
+      return null;
+    }
+
+    var setIds = _.pluck(sets, 'id');
+    var setBackends = _.uniq(_.pluck(sets, 'backend'));
+    if (setBackends.length > 1) {
+      showError("Currently all sets of the same type " +
+        "must be from the same backend");
+      return null;
+    }
+
     var queryParams = {};
-    queryParams.readsetIds = readsetIds.join(',');
-    queryParams.backend = readsetBackend;
-    queryParams.type = type;
+    queryParams.setIds = setIds.join(',');
+    queryParams.backend = setBackends[0];
     queryParams.sequenceName = currentSequence.name;
     queryParams.sequenceStart = parseInt(sequenceStart);
     queryParams.sequenceEnd = parseInt(sequenceEnd);
     return queryParams;
   };
 
-  var queryReads = function(sequenceStart, sequenceEnd) {
-    queryApi(sequenceStart, sequenceEnd, 'reads', setReads);
-  };
-
-  // TODO: Make this cleaner
-  var queryApi = function(sequenceStart, sequenceEnd, type, handler) {
-    var queryParams = makeQueryParams(sequenceStart, sequenceEnd, type);
+  var queryData = function(start, end) {
+    var readParams = makeQueryParams(start, end, READSET_TYPE);
+    var variantParams = makeQueryParams(start, end, CALLSET_TYPE);
 
     if (xhrTimeout) {
       clearTimeout(xhrTimeout);
     }
 
     xhrTimeout = setTimeout(function() {
-      callXhr('/api/reads', queryParams, handler);
+      if (readParams) {
+        callXhr('/api/reads', readParams, setReads);
+      } else {
+        setReads([]);
+      }
+      if (variantParams) {
+        callXhr('/api/variants', variantParams, setVariants);
+      } else {
+        setVariants([]);
+      }
     }, 500);
   };
 
-  var callXhr = function(url, queryParams, handler, opt_reads) {
+  var callXhr = function(url, queryParams, handler, opt_data) {
     spinner.style('display', 'block');
     $.getJSON(url, queryParams)
         .done(function(res) {
-          lastQueryParams = queryParams;
-          var reads = (opt_reads || []).concat(res.reads || []);
-          handler(reads);
+          var data = (opt_data || []).concat(res.reads || res.variants || []);
+          handler(data);
 
           if (res.nextPageToken) {
             queryParams['pageToken'] = res.nextPageToken;
-            callXhr(url, queryParams, handler, reads);
+            callXhr(url, queryParams, handler, data);
           } else {
             spinner.style('display', 'none');
           }
@@ -745,30 +875,28 @@ var readgraph = new function() {
         });
   };
 
-  this.hasReadset = function(id) {
-    return $.inArray(id, readsetIds) != -1;
-  };
+  this.getCurrentSequence = function() {
+    return currentSequence.name;
+  }
 
-  // TODO: Support multiple readsets
-  this.addReadset = function(backend, id, sequenceData) {
-    readsetBackend = backend;
-    readsetIds = [id];
-    if (readsetIds.length == 1) {
-      sequences = sequenceData;
-      updateSequences();
-      $('#chooseReadsetMessage').hide();
-      clearReads();
+  this.updateSets = function(setData) {
+    if (_.isEqual(setObjects, setData)) {
+      return;
     }
-  };
 
-  this.removeReadset = function(id) {
-    readsetIds = _.without(readsetIds, id);
-    if (readsetIds.length == 0) {
-      sequences = [];
-      updateSequences();
-      $('#chooseReadsetMessage').show();
+    setObjects = setData;
+    updateSequences();
+
+    if (setObjects.length == 0) {
+      $('#chooseSetMessage').show();
       $('#graph').hide();
       $('#jumpDiv').hide();
+
+      readDiv && readDiv.hide();
+      variantDiv && variantDiv.hide();
+
+    } else {
+      $('#chooseSetMessage').hide();
     }
-  }
+  };
 };

From 10a6669849f24b741f7c7859213a3437a018df7c Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Mon, 9 Jun 2014 12:55:17 -0700
Subject: [PATCH 03/42] Add missing return false

---
 main.html         | 2 +-
 static/js/main.js | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/main.html b/main.html
index 1022ae8..bde533e 100644
--- a/main.html
+++ b/main.html
@@ -107,7 +107,7 @@ <h4 class="modal-title">Select data to view</h4>
           <div class="form-group">
             <input type="text" class="form-control" placeholder="name" id="setName"/>
           </div>
-          <button type="submit" class="btn btn-primary" onclick="return searchSets(this)">Search</button>
+          <button type="submit" class="btn btn-primary" onclick="searchSets(this); return false;">Search</button>
         </form>
         <div>
           <ul class="nav nav-tabs">
diff --git a/static/js/main.js b/static/js/main.js
index 773204a..d9297e0 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -203,7 +203,6 @@ function searchSetsOfType(button, setType, backend, datasetId) {
       }).always(function() {
         button && button.button('reset');
       });
-  return false;
 }
 
 

From c49689d7dfcac16c8f1f2f272c8e66b5fb32c0a6 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Mon, 9 Jun 2014 13:00:54 -0700
Subject: [PATCH 04/42] Use actual callsets.get and variants.summary apis

---
 main.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/main.py b/main.py
index d178722..002921c 100644
--- a/main.py
+++ b/main.py
@@ -153,11 +153,17 @@ def get(self):
 
     # Single object response
     if use_callsets:
-      # TODO: Use GET /callsets/id and GET /variants/summary
-      response = {'id' : set_id, 'name' : set_id, 'contigs': [
-        {'name' : '22', 'length': 51304566}
-      ]}
-      self.response.write(json.dumps(response))
+      callset = self.get_content('%s/%s' % (set_type, set_id), method="GET")
+
+      # For callsets, we also load up the variant summary data to get
+      # the available contigs and lengths
+      dataset_id = callset['datasetId']
+      summary = self.get_content('variants/summary', method="GET",
+                                 params='datasetId=%s' % dataset_id)
+
+      callset['contigs'] = [{'name': b['contig'], 'length': b['upperBound']}
+                            for b in summary['contigBounds']]
+      self.response.write(json.dumps(callset))
       return
 
     self.write_content('%s/%s' % (set_type, set_id), method='GET')

From 04f61aec6f95df805ce1a466c4c5b10195f08206 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Mon, 9 Jun 2014 15:53:32 -0700
Subject: [PATCH 05/42] Add variant outlines which display at high zoom

---
 static/css/main.css    |  4 ++++
 static/js/readgraph.js | 47 ++++++++++++++++++++++++++++++++----------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/static/css/main.css b/static/css/main.css
index 37996d6..cdeecd8 100644
--- a/static/css/main.css
+++ b/static/css/main.css
@@ -151,6 +151,10 @@ svg {
   text-anchor: middle;
 }
 
+.variant .outline {
+  stroke: black;
+}
+
 .coverageSummary, .coverage {
   fill: none;
   stroke: black;
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index b0c68db..09d43d2 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -438,14 +438,18 @@ var readgraph = new function() {
 
     var reads = readGroup.selectAll(".read");
     var variants = readGroup.selectAll(".variant");
-    var outlines = reads.selectAll(".outline");
-    var letters = reads.selectAll(".letter");
+
+    var readOutlines = reads.selectAll(".outline");
+    var readLetters = reads.selectAll(".letter");
+    var variantOutlines = variants.selectAll(".outline");
+    var variantLetters = variants.selectAll(".letter");
 
     toggleVisibility(unsupportedMessage, summaryView || coverageView);
-    toggleVisibility(outlines, readView);
-    toggleVisibility(letters, baseView);
+    toggleVisibility(readOutlines, readView);
+    toggleVisibility(variantOutlines, readView);
+    toggleVisibility(readLetters, baseView);
+    toggleVisibility(variantLetters, baseView);
     toggleVisibility(positionIndicator, baseView);
-    toggleVisibility(variants, baseView);
 
     var sequenceStart = parseInt(x.domain()[0]);
     var sequenceEnd = parseInt(x.domain()[1]);
@@ -456,11 +460,19 @@ var readgraph = new function() {
 
     // TODO: Bring back coverage and summary views
     if (readView) {
-      outlines.attr("points", outlinePoints);
+      // Read outlines
+      readOutlines.attr("points", outlinePoints);
+
+      // Variant outlines
+      variantOutlines
+        .attr("x1", function(data) { return x(data.rx) + textWidth; })
+        .attr("x2", function(data) { return x(data.rx) + textWidth; })
+        .attr("y1", function(data) { return y(maxY - data.ry); })
+        .attr("y2", function(data) { return y(maxY - data.ry) + textHeight; });
 
     } else if (baseView) {
       // Read bases
-      letters.style('display', function(data, i) {
+      readLetters.style('display', function(data, i) {
             if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
               return 'none';
             } else {
@@ -496,7 +508,7 @@ var readgraph = new function() {
       }
 
       // Variants
-      variants.style('display', function(data, i) {
+      variantLetters.style('display', function(data, i) {
             if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
               return 'none';
             } else {
@@ -504,7 +516,7 @@ var readgraph = new function() {
             }
           })
           .attr("x", function(data, i) {
-            return x(data.rx);
+            return x(data.rx) + textWidth;
           })
           .attr("y", function(data, i) {
             return y(maxY - data.ry) + textHeight/2;
@@ -670,12 +682,25 @@ var readgraph = new function() {
     var variantDivs = readGroup.selectAll(".variant").data(data,
         function(data){ return data.id; });
 
-    variantDivs.enter().append("text")
+    variantDivs.enter().append("g")
         .attr('class', 'variant')
-        .text(function(data, i) { return data.genotype; })
         .on("mouseover", showVariant)
         .on("mouseout", deselectObject);
 
+    var outlines = variantDivs.selectAll('.outline')
+        .data(function(variant, i) { return [variant];});
+    outlines.enter().append('line').attr('class', 'outline');
+
+    var baseView = getScaleLevel() > 5;
+    if (baseView) {
+      var bases = variantDivs.selectAll(".letter")
+          .data(function(variant, i) { return [variant];});
+
+      bases.enter().append('text')
+          .attr('class', 'letter')
+          .text(function(data, i) { return data.genotype; });
+    }
+
     variantDivs.exit().remove();
 
     callsetTrackLength = maxCalls;

From 5a77c3bfe0d11f747feb1f5d3d88115df9eee648 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassiedoll@users.noreply.github.com>
Date: Fri, 13 Jun 2014 12:13:46 -0700
Subject: [PATCH 06/42] Update README.rst

---
 README.rst | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index e5ff6c2..38dad0c 100644
--- a/README.rst
+++ b/README.rst
@@ -54,13 +54,10 @@ If you want to pull in data from `Google Genomics API`_ you will need to set
 * First apply for access to the Genomics API by following the instructions at
   https://developers.google.com/genomics/
 
-* Then create a project in the
-  `Google Developers Console <https://console.developers.google.com>`_
-  or select an existing one.
+* Then create a `Genomics enabled project <https://console.developers.google.com/flows/enableapi?apiid=genomics>`_ 
+  in the Google Developers Console.
 
-* On the **APIs & auth** tab, select APIs and turn the Genomics API to ON
-
-* On the **Credentials** tab, click **create new key** under
+* Once you are redirected to the **Credentials** tab, click **create new key** under
   the Public API access section.
 
 * Select **Server key** in the dialog that pops up, and then click **Create**.

From 4dae088ec764be21d360d92dfae47b6d7ad1b51e Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Fri, 13 Jun 2014 13:47:16 -0700
Subject: [PATCH 07/42] Fix location jumping for non-ints

- also clean up some code duplication
---
 main.html              |  4 +-
 static/js/main.js      |  2 +-
 static/js/readgraph.js | 85 ++++++++++++++++++++++--------------------
 3 files changed, 47 insertions(+), 44 deletions(-)

diff --git a/main.html b/main.html
index bde533e..630f4f2 100644
--- a/main.html
+++ b/main.html
@@ -162,8 +162,8 @@ <h5 id="READSETTitle" style="display:none">Readsets</h5>
       </form>
     </div>
 
-    <div class="well infoDiv" style="display: none" id="variantDiv"></div>
-    <div class="well infoDiv" style="display: none" id="readDiv"></div>
+    <div class="infoDiv" style="display: none" id="variantDiv"></div>
+    <div class="infoDiv" style="display: none" id="readDiv"></div>
   </div>
 
   <div class="col-md-10">
diff --git a/static/js/main.js b/static/js/main.js
index d9297e0..64f6ebd 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -265,7 +265,7 @@ function switchToLocation(location) {
 }
 
 function updateUserLocation(location) {
-  switchToLocation(readgraph.getCurrentSequence() + ":" + location);
+  switchToLocation(readgraph.jumpGraph(location));
 }
 
 function handleHash() {
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 09d43d2..8907393 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -229,6 +229,9 @@ var readgraph = new function() {
   };
 
   var chrLocation = /^(.*):(\d*)$/;
+
+  // Jumps the graph to the given user-entered position
+  // Returns a bookmarkable position value
   this.jumpGraph = function(location) {
     var jumpResults = $("#jumpResults").empty();
 
@@ -236,14 +239,14 @@ var readgraph = new function() {
     if (chrLocation.test(location)) {
       var matches = chrLocation.exec(location);
       jumpToPosition(parseInt(matches[2].replace(/,/g, '')), matches[1], true);
-      return;
+      return location;
     }
 
     var position = parseInt(location.replace(/,/g, ''));
     // Numbered locations
     if (position > 0) {
       jumpToPosition(position, null, true);
-      return;
+      return currentSequence.name + ":" + position;
     }
 
     // Queried locations
@@ -281,6 +284,7 @@ var readgraph = new function() {
         $("#jumpResults .list-group-item").click();
       }
     });
+    return location;
   };
 
   var fuzzyFindSequence = function(chr) {
@@ -461,7 +465,7 @@ var readgraph = new function() {
     // TODO: Bring back coverage and summary views
     if (readView) {
       // Read outlines
-      readOutlines.attr("points", outlinePoints);
+      readOutlines.attr("points", readOutlinePoints);
 
       // Variant outlines
       variantOutlines
@@ -508,14 +512,7 @@ var readgraph = new function() {
       }
 
       // Variants
-      variantLetters.style('display', function(data, i) {
-            if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
-              return 'none';
-            } else {
-              return 'block';
-            }
-          })
-          .attr("x", function(data, i) {
+      variantLetters.attr("x", function(data, i) {
             return x(data.rx) + textWidth;
           })
           .attr("y", function(data, i) {
@@ -553,7 +550,7 @@ var readgraph = new function() {
     return points.join(' ');
   };
 
-  var outlinePoints = function(read, i) {
+  var readOutlinePoints = function(read, i) {
     var yTracksLength = y.domain()[0];
     var barHeight = Math.min(30, Math.max(2,
         (height - margin * 3) / yTracksLength - 5));
@@ -592,6 +589,22 @@ var readgraph = new function() {
   };
 
   // Hover details
+  var showObject = function(item, div, title, fields) {
+    div.empty().show();
+    closeButton().appendTo(div).click(function() {
+      div.hide();
+    });
+
+    $("<h4/>").text(title).appendTo(div);
+    var dl = $("<dl/>").addClass("dl-horizontal").appendTo(div);
+
+    $.each(fields, function(i, field) {
+      addField(dl, field[0], field[1]);
+    });
+
+    d3.select(item).classed("selected", true);
+  };
+
   var addField = function(dl, title, field) {
     if (field) {
       $("<dt/>").text(title).appendTo(dl);
@@ -599,40 +612,33 @@ var readgraph = new function() {
     }
   };
 
-  var showRead = function(read, i) {
-    readDiv.empty().show();
-
-    $("<h4/>").text("Read: " + read.name).appendTo(readDiv);
-    var dl = $("<dl/>").addClass("dl").appendTo(readDiv);
-
-    addField(dl, "Position", read.position);
-    addField(dl, "Length", read.length);
-    addField(dl, "Mate position", read.matePosition);
-    addField(dl, "Mapping quality", read.mappingQuality);
-    addField(dl, "Cigar", read.cigar);
-
-    d3.select(this).classed("selected", true);
+  var showRead = function(read) {
+    showObject(this, readDiv, "Read: " + read.name, [
+      ["Position", read.position],
+      ["Length", read.length],
+      ["Mate position", read.matePosition],
+      ["Mapping quality", read.mappingQuality],
+      ["Cigar", read.cigar]
+    ]);
   };
 
   var showVariant = function(data) {
-    variantDiv.empty().show();
     var variant = data.variant;
     var call = variant.calls[data.callIndex];
 
-    $("<h4/>").text("Variant: " + variant.names.join(", "))
-      .appendTo(variantDiv);
-    var dl = $("<dl/>").addClass("dl").appendTo(variantDiv);
-
-    addField(dl, "Callset name", call.callsetName);
-    addField(dl, "Position", variant.position);
-
-    d3.select(this).classed("selected", true);
+    showObject(this, variantDiv, "Variant: " + variant.names.join(", "), [
+      ["Callset name", call.callsetName],
+      ["Position", variant.position]
+    ]);
   };
 
   var deselectObject = function(read, i) {
     d3.select(this).classed("selected", false);
   };
 
+
+  // D3 object creation
+
   var setYOrder = function(read, yOrder) {
     read.yOrder = yOrder;
 
@@ -836,6 +842,9 @@ var readgraph = new function() {
     updateDisplay(true);
   };
 
+
+  // Data loading
+
   var makeQueryParams = function(sequenceStart, sequenceEnd, type) {
     var sets = _.where(setObjects, {type: type});
     if (sets.length == 0) {
@@ -900,10 +909,6 @@ var readgraph = new function() {
         });
   };
 
-  this.getCurrentSequence = function() {
-    return currentSequence.name;
-  }
-
   this.updateSets = function(setData) {
     if (_.isEqual(setObjects, setData)) {
       return;
@@ -916,9 +921,7 @@ var readgraph = new function() {
       $('#chooseSetMessage').show();
       $('#graph').hide();
       $('#jumpDiv').hide();
-
-      readDiv && readDiv.hide();
-      variantDiv && variantDiv.hide();
+      $('.infoDiv').hide();
 
     } else {
       $('#chooseSetMessage').hide();

From b750a669e30131dc7173b9a29f40f4c82eab5db9 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassiedoll@users.noreply.github.com>
Date: Tue, 17 Jun 2014 10:25:33 -0700
Subject: [PATCH 08/42] Fix GA4GH API name

---
 main.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/main.html b/main.html
index 630f4f2..a57f9d7 100644
--- a/main.html
+++ b/main.html
@@ -51,7 +51,7 @@ <h4 class="modal-title">About GABrowse</h4>
       </div>
       <div class="modal-body">
         <p class="lead">GABrowse is a sample application designed to demonstrate the capabilities of the
-        <a href="http://ga4gh.github.io/apis/reads/v0.1/">GA4GH Reads API v0.1</a>.</p>
+        <a href="http://ga4gh.github.io/apis/reads/v0.1/">GA4GH API v0.1</a>.</p>
 
         <p>Currently, you can view data from Google, NCBI and EBI.</p>
 

From 67575513c0eb5fb10ccdc75834a81ea45eb83e0a Mon Sep 17 00:00:00 2001
From: jbingham <jb@jbingham.com>
Date: Fri, 20 Jun 2014 09:26:49 -0700
Subject: [PATCH 09/42] Update README.rst

More details on genome browser features.
---
 README.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 38dad0c..397ca1e 100644
--- a/README.rst
+++ b/README.rst
@@ -4,9 +4,11 @@ api-client-python
 Getting started
 ---------------
 
-This Python client provides a simple web application which fetches data from the
+This Python client demonstrates a simple web-based genome browser that fetches data from the 
 `Google Genomics API`_, the NCBI Genomics API or the Local Readstore through a web
-interface.
+interface, and displays a pileup of reads with support for zooming and basic navigation and search.
+
+You can try out the sample genome browser, called GABrowse, now by going to https://gabrowser.appspot.com.
 
 It can be run with app engine or without.
 See `the docs <http://google-genomics.readthedocs.org/en/latest/api-client-python>`_

From c888f51f5dea225ac841d584047536353b9415f5 Mon Sep 17 00:00:00 2001
From: jbingham <jb@jbingham.com>
Date: Fri, 20 Jun 2014 09:27:33 -0700
Subject: [PATCH 10/42] Update README.rst

Fix typo in url.
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 397ca1e..f2aa3f0 100644
--- a/README.rst
+++ b/README.rst
@@ -8,7 +8,7 @@ This Python client demonstrates a simple web-based genome browser that fetches d
 `Google Genomics API`_, the NCBI Genomics API or the Local Readstore through a web
 interface, and displays a pileup of reads with support for zooming and basic navigation and search.
 
-You can try out the sample genome browser, called GABrowse, now by going to https://gabrowser.appspot.com.
+You can try out the sample genome browser, called GABrowse, now by going to https://gabrowse.appspot.com.
 
 It can be run with app engine or without.
 See `the docs <http://google-genomics.readthedocs.org/en/latest/api-client-python>`_

From 725bf8b068956b71dfbc9eacc2509928f37881e2 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Fri, 20 Jun 2014 10:08:58 -0700
Subject: [PATCH 11/42] Stop complaining when the seq name doesn't match

- so when you switch between readsets of different types, you no longer
get a spurious error message
---
 static/js/readgraph.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 8907393..8db0500 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -303,8 +303,6 @@ var readgraph = new function() {
       // Update our sequence
       var sequence = fuzzyFindSequence(chr);
       if (!sequence) {
-        showError('This data doesn\'t have the sequence ' + chr +
-          '. Please try a different position.');
         return;
       }
 

From be32c14741739c855297d152fcf0723be6d218dd Mon Sep 17 00:00:00 2001
From: cassiedoll <cassandradoll@gmail.com>
Date: Fri, 20 Jun 2014 10:17:14 -0700
Subject: [PATCH 12/42] Update todos

---
 README.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index f2aa3f0..961d5e0 100644
--- a/README.rst
+++ b/README.rst
@@ -151,12 +151,11 @@ from any API provider. It deploys on App Engine (to meet the
 
 Awesome possible features include:
 
-* Incorporating variant data into the UI.
 * Add more information to the read display (show inserts, highlight mismatches
   against the reference, etc)
 * Possibly cleaning up the js code to be more plugin friendly - so that pieces
   could be shared and reused (d3 library? jquery plugin?)
 * Staying up to date on API changes (readset searching now has pagination, etc)
-* Better searching of Snpedia (or another provider)
+* Better searching of Snpedia (or another provider - EBI?)
 * Other enhancement ideas are very welcome
 * (for smaller/additional tasks see the GitHub issues)

From 5274b4f8b9aa8605d663e5a458a21fb6d50a65f6 Mon Sep 17 00:00:00 2001
From: cassiedoll <cassiedoll@users.noreply.github.com>
Date: Tue, 1 Jul 2014 10:00:31 -0700
Subject: [PATCH 13/42] Update README.rst

---
 README.rst | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 961d5e0..66ef16e 100644
--- a/README.rst
+++ b/README.rst
@@ -53,10 +53,7 @@ Enabling the Google API
 If you want to pull in data from `Google Genomics API`_ you will need to set
 ``API_KEY`` in ``main.py`` to a valid Google API key.
 
-* First apply for access to the Genomics API by following the instructions at
-  https://developers.google.com/genomics/
-
-* Then create a `Genomics enabled project <https://console.developers.google.com/flows/enableapi?apiid=genomics>`_ 
+* First create a `Genomics enabled project <https://console.developers.google.com/flows/enableapi?apiid=genomics>`_ 
   in the Google Developers Console.
 
 * Once you are redirected to the **Credentials** tab, click **create new key** under

From fd7f680eb8442fb69f1cb50eac9f4af147a5b340 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 7 Jul 2014 15:45:09 -0400
Subject: [PATCH 14/42] Add quick and dirty load performance instrumentation

Instruments the client-side XHRs to add Chrome devtools timeline events and
console timing for loads.  Also adds a log message to the python code to
track timing and sizes of loads.
---
 main.py                |  7 ++++++-
 static/js/readgraph.js | 25 ++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/main.py b/main.py
index 002921c..0d96ce4 100644
--- a/main.py
+++ b/main.py
@@ -23,6 +23,7 @@
 GOOGLE_API_KEY = None
 
 import httplib2
+import time
 import jinja2
 import json
 import logging
@@ -98,10 +99,13 @@ def get_base_api_url(self):
     return SUPPORTED_BACKENDS[self.get_backend()]['url']
 
   def get_content(self, path, method='POST', body=None, params=''):
+    uri= self.get_base_api_url() % (path, params)
+    startTime = time.clock()
     response, content = http.request(
-      uri= self.get_base_api_url() % (path, params),
+      uri,
       method=method, body=json.dumps(body) if body else None,
       headers={'Content-Type': 'application/json; charset=UTF-8'})
+    logging.info('get_content {}: {}kb {}s'.format(uri, len(content)/1024, time.clock() - startTime))
 
     try:
       content = json.loads(content)
@@ -116,6 +120,7 @@ def get_content(self, path, method='POST', body=None, params=''):
         raise ApiException(content['error']['message'])
       else:
         raise ApiException('Something went wrong with the API call!')
+
     return content
 
   def write_content(self, path, method='POST', body=None):
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 8db0500..2230910 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -47,6 +47,8 @@ var readgraph = new function() {
   var readTrackLength = 0;
   var callsetTrackLength = 0;
 
+  var spinnerShown = false;
+
   // Dom elements
   var svg, axisGroup, readGroup, readDiv, variantDiv, spinner = null;
   var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
@@ -888,8 +890,25 @@ var readgraph = new function() {
     }, 500);
   };
 
+  var showSpinner = function(show) {
+    if (show == spinnerShown)
+      return;
+    spinnerShown = show;
+    spinner.style('display', show ? 'block' : 'none');
+    if ('time' in console) {
+      var timeLabel = 'readgraph loading';
+      if (show)
+        console.time(timeLabel);
+      else
+        console.timeEnd(timeLabel);
+    }
+    if ('timeStamp' in console) {
+      console.timeStamp('readgraph load ' + (show ? 'start' : 'finish'));
+    }
+  }
+
   var callXhr = function(url, queryParams, handler, opt_data) {
-    spinner.style('display', 'block');
+    showSpinner(true);
     $.getJSON(url, queryParams)
         .done(function(res) {
           var data = (opt_data || []).concat(res.reads || res.variants || []);
@@ -899,11 +918,11 @@ var readgraph = new function() {
             queryParams['pageToken'] = res.nextPageToken;
             callXhr(url, queryParams, handler, data);
           } else {
-            spinner.style('display', 'none');
+            showSpinner(false);
           }
         })
         .fail(function() {
-          spinner.style('display', 'none');
+          showSpinner(false);
         });
   };
 

From 15f051856ecc03eae1143854721fb36ddcb32cbe Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 7 Jul 2014 16:11:27 -0400
Subject: [PATCH 15/42] Read search optimization: request only the fields we
 use

This omits, for example, the unused alignedBases and tags field of reads which
can be quite large.  In one example of navigating a 1000 genomes readset at
the 'reads' scale, this reduced data transfer volume for a typical navigation
from 400kB to 150kB, and client load time from about 2s to 1.5s.
---
 main.py                | 10 +++++++---
 static/js/readgraph.js |  5 +++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/main.py b/main.py
index 0d96ce4..41315f1 100644
--- a/main.py
+++ b/main.py
@@ -123,8 +123,8 @@ def get_content(self, path, method='POST', body=None, params=''):
 
     return content
 
-  def write_content(self, path, method='POST', body=None):
-    self.response.write(json.dumps(self.get_content(path, method, body)))
+  def write_content(self, path, method='POST', body=None, params=''):
+    self.response.write(json.dumps(self.get_content(path, method, body, params)))
 
 
 class SetSearchHandler(BaseRequestHandler):
@@ -183,10 +183,14 @@ def get(self):
       'sequenceStart': max(0, int(self.request.get('sequenceStart'))),
       'sequenceEnd': int(self.request.get('sequenceEnd')),
     }
+    fields = self.request.get('fields')
+    params = ''
+    if fields:
+      params = 'fields=%s' % fields
     pageToken = self.request.get('pageToken')
     if pageToken:
       body['pageToken'] = pageToken
-    self.write_content('reads/search', body=body)
+    self.write_content('reads/search', body=body, params=params)
 
 
 class VariantSearchHandler(BaseRequestHandler):
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 2230910..dd94bb4 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -865,6 +865,11 @@ var readgraph = new function() {
     queryParams.sequenceName = currentSequence.name;
     queryParams.sequenceStart = parseInt(sequenceStart);
     queryParams.sequenceEnd = parseInt(sequenceEnd);
+    
+    if (type == READSET_TYPE) {
+      queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,baseQuality,cigar,originalBases)'
+    }
+
     return queryParams;
   };
 

From 27dfd4fdf004e87f0b0c74af85b0a5811d0519cb Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 7 Jul 2014 17:07:24 -0400
Subject: [PATCH 16/42] Read optimization: only request per-base fields at
 base-level zoom

originalBases and baseQuality are the only fields of a read which
scale directly with the size of a read.  When at a read-level zoom
scale we shouldn't request and manipulate all this data.  This
change has a minor impact on the API in that hovering over a read
no longer displays the base at that position, but at read-level
scales that was pretty useless anyway (since you couldn't precisely
position the mouse cursor at each position).

For a simple 1000 genome's scenario this reduces the data transfer
volume for a simple navigation from about 150kb to about 60kb.
This also reduces the amount of heap allocations (and hence the GC
pressure) on each such navigation from about 7MB to around 600kB.
The reduced memory usage should be particularly important on
low-power and low-memory devices like phones.  Anecdotally I've
seen some very long (>1s) GC pauses when doing navigation at
read-level, so this should help improve things dramatically.
---
 static/js/readgraph.js | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index dd94bb4..189d00e 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -751,9 +751,12 @@ var readgraph = new function() {
         });
       };
 
-      var bases = read.originalBases.split('');
+      var bases;
+      if (read.originalBases)
+        bases = read.originalBases.split('');
       var baseIndex = 0;
       var matches = read.cigar.match(cigarMatcher);
+      read.length = 0;
 
       for (var m = 0; m < matches.length; m++) {
         var match = matches[m];
@@ -769,7 +772,9 @@ var readgraph = new function() {
           case 'N':
             // Deletions get placeholders inserted
             for (var b = 0; b < baseCount; b++) {
-              addLetter(baseType, '-', 100);
+              if (bases)
+                addLetter(baseType, '-', 100);
+              read.length++;
             }
             break;
           case 'S': // TODO: Reveal this skipped data somewhere
@@ -781,15 +786,16 @@ var readgraph = new function() {
           case '=':
             // Matches and insertions get displayed
             for (var j = 0; j < baseCount; j++) {
-              addLetter(baseType, bases[baseIndex],
+              if (bases)
+                addLetter(baseType, bases[baseIndex],
                   read.baseQuality.charCodeAt(baseIndex) - 33);
               baseIndex++;
+              read.length++;
             }
             break;
         }
       }
 
-      read.length = read.readPieces.length;
       read.end = read.position + read.length;
       // The 5th flag bit indicates this read is reversed
       read.reverse = (read.flags >> 4) % 2 == 1;
@@ -867,7 +873,9 @@ var readgraph = new function() {
     queryParams.sequenceEnd = parseInt(sequenceEnd);
     
     if (type == READSET_TYPE) {
-      queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,baseQuality,cigar,originalBases)'
+      var baseView = getScaleLevel() > 5;
+      var baseFields = baseView ? ',originalBases,baseQuality' : '';
+      queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,cigar' + baseFields + ')';
     }
 
     return queryParams;

From 9ff2ce086cd926de2eea49bf63925050a93791e5 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 7 Jul 2014 17:15:19 -0400
Subject: [PATCH 17/42] JS style: add braces to one-line if statements

---
 static/js/readgraph.js | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 189d00e..88350a0 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -752,8 +752,9 @@ var readgraph = new function() {
       };
 
       var bases;
-      if (read.originalBases)
+      if (read.originalBases) {
         bases = read.originalBases.split('');
+      }
       var baseIndex = 0;
       var matches = read.cigar.match(cigarMatcher);
       read.length = 0;
@@ -772,8 +773,9 @@ var readgraph = new function() {
           case 'N':
             // Deletions get placeholders inserted
             for (var b = 0; b < baseCount; b++) {
-              if (bases)
+              if (bases) {
                 addLetter(baseType, '-', 100);
+              }
               read.length++;
             }
             break;
@@ -786,9 +788,10 @@ var readgraph = new function() {
           case '=':
             // Matches and insertions get displayed
             for (var j = 0; j < baseCount; j++) {
-              if (bases)
+              if (bases) {
                 addLetter(baseType, bases[baseIndex],
                   read.baseQuality.charCodeAt(baseIndex) - 33);
+              }
               baseIndex++;
               read.length++;
             }
@@ -904,16 +907,18 @@ var readgraph = new function() {
   };
 
   var showSpinner = function(show) {
-    if (show == spinnerShown)
+    if (show == spinnerShown) {
       return;
+    }
     spinnerShown = show;
     spinner.style('display', show ? 'block' : 'none');
     if ('time' in console) {
       var timeLabel = 'readgraph loading';
-      if (show)
+      if (show) {
         console.time(timeLabel);
-      else
+      } else {
         console.timeEnd(timeLabel);
+      }
     }
     if ('timeStamp' in console) {
       console.timeStamp('readgraph load ' + (show ? 'start' : 'finish'));

From 9abbe3a3913b83b6fb8c1a2016a1f0d5f4ca0a88 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 11:32:15 -0400
Subject: [PATCH 18/42] Tweak read performance instrumentation

Ensure we're really timing everything we're doing for handling a read request.
It seems surprising how fast this is but how slow things can be from the JS
side.
---
 main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/main.py b/main.py
index 41315f1..3bce0c3 100644
--- a/main.py
+++ b/main.py
@@ -105,7 +105,7 @@ def get_content(self, path, method='POST', body=None, params=''):
       uri,
       method=method, body=json.dumps(body) if body else None,
       headers={'Content-Type': 'application/json; charset=UTF-8'})
-    logging.info('get_content {}: {}kb {}s'.format(uri, len(content)/1024, time.clock() - startTime))
+    contentLen = len(content)
 
     try:
       content = json.loads(content)
@@ -121,6 +121,7 @@ def get_content(self, path, method='POST', body=None, params=''):
       else:
         raise ApiException('Something went wrong with the API call!')
 
+    logging.info('get_content {}: {}kb {}s'.format(uri, contentLen, time.clock() - startTime))
     return content
 
   def write_content(self, path, method='POST', body=None, params=''):

From f2595e3b7a1566b07b92572ae4eb2429dd3322d3 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 11:33:56 -0400
Subject: [PATCH 19/42] Upd data immediate after a navigation

Rather than add a 500ms delay, don't ever update data during a pan/zoom
operation.  But once the operation is complete update the data immediately.
This substantially improves the feeling of responsiveness.
---
 static/js/readgraph.js | 55 ++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 88350a0..e44195e 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -42,7 +42,6 @@ var readgraph = new function() {
   var mergedSequences = [];
   var currentSequence = null;
   var readStats = {}; // Map from position to stat information
-  var xhrTimeout = null;
 
   var readTrackLength = 0;
   var callsetTrackLength = 0;
@@ -76,6 +75,8 @@ var readgraph = new function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
   };
 
+  // Called at high frequency for any navigation of the UI (not just zooming).
+  // Should only do low-latency operations.
   var handleZoom = function() {
     var tx = zoom.translate()[0];
     // TODO: This isn't strict enough
@@ -88,6 +89,18 @@ var readgraph = new function() {
     d3.select('.zoomLevel').attr('y', (6 - getScaleLevel()) * 24 + 38);
     updateDisplay();
   };
+  
+  // Called when a navigation operation has completed.  This is the place to
+  // do more expensive operations than simple UI updates.  Should not be
+  // called frequently.
+  var handleZoomEnd = function() {
+    handleZoom();
+    if (getScaleLevel() >= 4) {
+      var sequenceStart = parseInt(x.domain()[0]);
+      var sequenceEnd = parseInt(x.domain()[1]);
+      queryData(sequenceStart, sequenceEnd);
+    }
+  } 
 
   var moveToSequencePosition = function(position) {
     position = Math.max(0, position);
@@ -96,7 +109,7 @@ var readgraph = new function() {
     var newX = x(position);
     newX = zoom.translate()[0] - newX + width / 2;
     zoom.translate([newX, 0]);
-    handleZoom();
+    handleZoomEnd();
   };
 
   var setupRun = false;
@@ -199,11 +212,11 @@ var readgraph = new function() {
       newZoom = Math.min(maxZoom, newZoom);
       zoom.scale(newZoom);
 
-      handleZoom();
+      handleZoomEnd();
       moveToSequencePosition(middleX);
     };
 
-    zoom = d3.behavior.zoom().size([width, height]).on("zoom", handleZoom);
+    zoom = d3.behavior.zoom().size([width, height]).on("zoom", handleZoom).on("zoomend", handleZoomEnd);;
     svg.call(zoom);
 
     // Zoom background
@@ -431,7 +444,7 @@ var readgraph = new function() {
     $('#jumpDiv').show();
   };
 
-  var updateDisplay = function(opt_skipDataQuery) {
+  var updateDisplay = function() {
     var maxY = updateHeight();
 
     var scaleLevel = getScaleLevel();
@@ -458,10 +471,6 @@ var readgraph = new function() {
     var sequenceStart = parseInt(x.domain()[0]);
     var sequenceEnd = parseInt(x.domain()[1]);
 
-    if (!opt_skipDataQuery && (readView || baseView)) {
-      queryData(sequenceStart, sequenceEnd);
-    }
-
     // TODO: Bring back coverage and summary views
     if (readView) {
       // Read outlines
@@ -710,7 +719,7 @@ var readgraph = new function() {
     variantDivs.exit().remove();
 
     callsetTrackLength = maxCalls;
-    updateDisplay(true);
+    updateDisplay();
   };
 
   var setReads = function(reads) {
@@ -848,7 +857,7 @@ var readgraph = new function() {
           .text(function(data, i) { return data.letter; });
     }
     reads.exit().remove();
-    updateDisplay(true);
+    updateDisplay();
   };
 
 
@@ -888,22 +897,16 @@ var readgraph = new function() {
     var readParams = makeQueryParams(start, end, READSET_TYPE);
     var variantParams = makeQueryParams(start, end, CALLSET_TYPE);
 
-    if (xhrTimeout) {
-      clearTimeout(xhrTimeout);
+    if (readParams) {
+      callXhr('/api/reads', readParams, setReads);
+    } else {
+      setReads([]);
+    }
+    if (variantParams) {
+      callXhr('/api/variants', variantParams, setVariants);
+    } else {
+      setVariants([]);
     }
-
-    xhrTimeout = setTimeout(function() {
-      if (readParams) {
-        callXhr('/api/reads', readParams, setReads);
-      } else {
-        setReads([]);
-      }
-      if (variantParams) {
-        callXhr('/api/variants', variantParams, setVariants);
-      } else {
-        setVariants([]);
-      }
-    }, 500);
   };
 
   var showSpinner = function(show) {

From a2aa7533f5b3ecd2f718443c3665868e1c55bba8 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 12:20:38 -0400
Subject: [PATCH 20/42] Get Google API key from a separate file instead of
 directly in main.py

This makes hacking on main.py without checking in an API key less of a hassle.
---
 README.rst | 10 +++++-----
 main.py    | 36 +++++++++++++++++++-----------------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/README.rst b/README.rst
index 66ef16e..6f866c7 100644
--- a/README.rst
+++ b/README.rst
@@ -50,8 +50,8 @@ Then install the required dependencies and run the ``localserver.py`` file::
 Enabling the Google API
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-If you want to pull in data from `Google Genomics API`_ you will need to set
-``API_KEY`` in ``main.py`` to a valid Google API key.
+If you want to pull in data from `Google Genomics API`_ you will need to store a valid
+Google API key into a file named ``google_api_key.txt``.
 
 * First create a `Genomics enabled project <https://console.developers.google.com/flows/enableapi?apiid=genomics>`_ 
   in the Google Developers Console.
@@ -63,10 +63,10 @@ If you want to pull in data from `Google Genomics API`_ you will need to set
   (You don't need to enter anything in the text box)
 
 * Copy the **API key** field value that now appears in the Public API access
-  section into the top of ``main.py``. It should look something like this::
-
-    API_KEY = "abcdef12345abcdef"
+  section into a new file named ``google_api_key.txt`` in the same directory as ``main.py``.
+  The file should consist of a single undecorated line like this::
 
+    abcdef12345abcdef
 
 Note: You can also reuse an existing API key if you have one.
 Just make sure the Genomics API is turned on.
diff --git a/main.py b/main.py
index 3bce0c3..fa76373 100644
--- a/main.py
+++ b/main.py
@@ -18,10 +18,6 @@
 - and it provides a simple set of apis to the javascript
 """
 
-# If this is set to a valid API key string, then the Google
-# backend will be enabled.
-GOOGLE_API_KEY = None
-
 import httplib2
 import time
 import jinja2
@@ -40,6 +36,7 @@
     autoescape=True,
     extensions=['jinja2.ext.autoescape'])
 
+
 # TODO: Dataset information should come from the list datasets api call
 SUPPORTED_BACKENDS = {
   'NCBI' : {'name': 'NCBI',
@@ -53,19 +50,23 @@
              'datasets': {'All': ''}},
 }
 
-if GOOGLE_API_KEY:
-  # Google requires a valid API key
-  SUPPORTED_BACKENDS['GOOGLE'] = {
-    'name': 'Google',
-    'url': 'https://www.googleapis.com/genomics/v1beta/%s?key='
-           + GOOGLE_API_KEY + '&%s',
-    'supportsNameFilter': True,
-    #'supportsCallsets': True,
-    'datasets': {'1000 Genomes': '376902546192',
-                 'DREAM SMC Challenge': '337315832689',
-                 'PGP': '383928317087',
-                 'Simons Foundation' : '461916304629'}
-  }
+# Google requires a valid API key.  If the file 'google_api_key.txt' exists
+# then the Google API will be enabled.
+google_api_key_file = os.path.join(os.path.dirname(__file__), 'google_api_key.txt')
+if os.path.isfile(google_api_key_file):
+  with open(google_api_key_file, 'r') as file:
+    api_key = file.readline().strip()
+    SUPPORTED_BACKENDS['GOOGLE'] = {
+      'name': 'Google',
+      'url': 'https://www.googleapis.com/genomics/v1beta/%s?key='
+             + api_key + '&%s',
+      'supportsNameFilter': True,
+      #'supportsCallsets': True,
+      'datasets': {'1000 Genomes': '376902546192',
+                   'DREAM SMC Challenge': '337315832689',
+                   'PGP': '383928317087',
+                   'Simons Foundation' : '461916304629'}
+    }
 
 
 class ApiException(Exception):
@@ -110,6 +111,7 @@ def get_content(self, path, method='POST', body=None, params=''):
     try:
       content = json.loads(content)
     except ValueError:
+      logging.error('while requesting {}'.format(uri))
       logging.error('non-json api content %s' % content[:1000])
       raise ApiException('The API returned invalid JSON')
 

From 365e1b558ff035c75c3d10431c6fb46294efe2b4 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 13:35:19 -0400
Subject: [PATCH 21/42] Prevent flash of white when switching from read view to
 base view

While we're waiting for bases to load we should show the reads we have
rather than showing nothing.  This makes zooming more fluid.
---
 static/js/readgraph.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index e44195e..3601c83 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -461,6 +461,12 @@ var readgraph = new function() {
     var variantOutlines = variants.selectAll(".outline");
     var variantLetters = variants.selectAll(".letter");
 
+    // If we're trying to display bases but don't yet have the letters data,
+    // then just show the reads for now.  This ensures a smooth zooming experience.
+    if (baseView && readLetters.length > 0 && readLetters.every(function(r) { return r.length == 0 })) {
+      baseView = false;
+      readView = true;
+    }
     toggleVisibility(unsupportedMessage, summaryView || coverageView);
     toggleVisibility(readOutlines, readView);
     toggleVisibility(variantOutlines, readView);

From 3297b5b03217a148a16cdd7edb22f5f85535e11a Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 14:59:21 -0400
Subject: [PATCH 22/42] Use fixed-height tracks stacked vertically

Rather than space the tracks out into the available space, stack them up along
the bottom with a fixed height each.  This is the first tiny step in a larger UI
overhaul to enable fluid navigation.  Once a given read is in view it should
remain in the same relative position when panning left and right.
---
 static/js/readgraph.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 3601c83..bcc9959 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -53,9 +53,9 @@ var readgraph = new function() {
   var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
 
   var updateHeight = function() {
-    var totalTracks = readTrackLength + callsetTrackLength;
-    height = totalTracks * textHeight + 100;
+    height = (readTrackLength + callsetTrackLength) * textHeight + 100;
     height = Math.max(height, 450);
+    var totalTracks = (height - 100) / textHeight;
 
     y.range([margin, height - margin*2]).domain([totalTracks, -1]);
     $('#graph').height(height);

From e6997ee1bd50f865e433d555dfc3aa262af4ef90 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 15:09:19 -0400
Subject: [PATCH 23/42] Only show the about box on the first pageload in a
 session

It's annoying that reloading the page displays the about box.  Instead lets
show it automatically only once per session (i.e. until the tab is closed).
Users can of course still click on the about link anytime.
---
 static/js/main.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/static/js/main.js b/static/js/main.js
index 64f6ebd..e2bba4a 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -285,10 +285,13 @@ function handleHash() {
 }
 
 
-// Show the about popup when the page loads, read the hash,
+// Show the about popup when the page loads the first time, read the hash,
 // and prep the initial set search
 $(document).ready(function() {
-  $('#about').modal('show');
+  if (!sessionStorage.getItem('about-shown')) {
+    sessionStorage.setItem('about-shown', true);
+    $('#about').modal('show');
+  }
 
   $(document).ajaxError(function(e, xhr) {
     showError('Sorry, the api request failed for some reason. ' +

From 0cf25021dee30ad486ebf4391c4e3074e00c60f6 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 15:22:44 -0400
Subject: [PATCH 24/42] Fix stupid transfer logging bug

Somewhere along the line I lost the /1024 I had for reporting kb transferred
---
 main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/main.py b/main.py
index fa76373..806af7e 100644
--- a/main.py
+++ b/main.py
@@ -123,7 +123,7 @@ def get_content(self, path, method='POST', body=None, params=''):
       else:
         raise ApiException('Something went wrong with the API call!')
 
-    logging.info('get_content {}: {}kb {}s'.format(uri, contentLen, time.clock() - startTime))
+    logging.info('get_content {}: {}kb {}s'.format(uri, contentLen/1024, time.clock() - startTime))
     return content
 
   def write_content(self, path, method='POST', body=None, params=''):

From 02f36385c97eb0717d9bfb2150991eeea9207dc5 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Tue, 8 Jul 2014 23:55:51 -0400
Subject: [PATCH 25/42] Overhaul read management to support incremental updates

For smooth navigation we don't want individual reads changing vertical position,
or ideally not see whitespace getting filled by content after loading.

This CL overhauls how reads are managed to use a similar approach to the tile
cache in mapping applications.  We have a read cache for data that is nearby
the view, can quickly update the view (pan/scale) to show other elements
of the cache, and asychronously update the cache to add/remove items that
have become (or are no longer) "nearby".

This CL also limits the SVG elements that are created to just those that
need to be displayed at any given time.  Data sets with deep coverage
(or very long reads) would cause a huge number of text nodes, which we'd
then update the style of every frame.  Now we create/remove nodes as
needed, never creating letter nodes just to have them be hidden.
---
 static/js/readgraph.js | 245 ++++++++++++++++++++++++-----------------
 1 file changed, 141 insertions(+), 104 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index bcc9959..37f1f2e 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -48,6 +48,13 @@ var readgraph = new function() {
 
   var spinnerShown = false;
 
+  var readCache = {
+      desireBases: false,
+      desiredStart: 0,
+      desiredEnd: 0,
+      readsById: {}
+  };
+  
   // Dom elements
   var svg, axisGroup, readGroup, readDiv, variantDiv, spinner = null;
   var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
@@ -74,6 +81,11 @@ var readgraph = new function() {
   var getScaleLevel = function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
   };
+  
+  // Return whether two sequences overlap at all
+  var overlaps = function(start1, end1, start2, end2) {
+    return start1 < end2 && end1 > start2; 
+  }
 
   // Called at high frequency for any navigation of the UI (not just zooming).
   // Should only do low-latency operations.
@@ -95,10 +107,11 @@ var readgraph = new function() {
   // called frequently.
   var handleZoomEnd = function() {
     handleZoom();
-    if (getScaleLevel() >= 4) {
+    var scaleLevel = getScaleLevel();
+    if (scaleLevel >= 4) {
       var sequenceStart = parseInt(x.domain()[0]);
       var sequenceEnd = parseInt(x.domain()[1]);
-      queryData(sequenceStart, sequenceEnd);
+      ensureReadsCached(sequenceStart, sequenceEnd, scaleLevel > 5);
     }
   } 
 
@@ -444,65 +457,81 @@ var readgraph = new function() {
     $('#jumpDiv').show();
   };
 
+  // Update the UI for the current position.  Called frequently during a drag
+  // or zoom operation, and so this must be very fast (<16ms for 60fps dragging).
+  // This shouldn't create or manipulate any SVG/DOM objects for things not being
+  // displayed (eg. offscreen letters).
   var updateDisplay = function() {
     var maxY = updateHeight();
-
+    
     var scaleLevel = getScaleLevel();
     var summaryView = scaleLevel < 2;
     var coverageView = scaleLevel == 2 || scaleLevel == 3;
     var readView = scaleLevel == 4 || scaleLevel == 5;
     var baseView = scaleLevel > 5;
 
-    var reads = readGroup.selectAll(".read");
+    var sequenceStart = parseInt(x.domain()[0]);
+    var sequenceEnd = parseInt(x.domain()[1]);
+
+    var readsInView = d3.values(readCache.readsById).filter(function(read) {
+      return overlaps(read.position, read.end, sequenceStart, sequenceEnd);
+    });
+    var reads = readGroup.selectAll(".read").data(readsInView,
+      function(read) { return read.id; });
+    reads.enter().append("g")
+      .attr('class', 'read')
+      .on("mouseover", showRead)
+      .on("mouseout", deselectObject);
+    reads.exit().remove();
+
     var variants = readGroup.selectAll(".variant");
 
-    var readOutlines = reads.selectAll(".outline");
+    var readOutlines = reads.selectAll(".outline")
+      .data(function(read, i) { return [read];});
+    readOutlines.enter().append('polygon')
+      .attr('class', 'outline');
+    readOutlines.exit().remove();
+    
     var readLetters = reads.selectAll(".letter");
+
     var variantOutlines = variants.selectAll(".outline");
     var variantLetters = variants.selectAll(".letter");
 
-    // If we're trying to display bases but don't yet have the letters data,
-    // then just show the reads for now.  This ensures a smooth zooming experience.
-    if (baseView && readLetters.length > 0 && readLetters.every(function(r) { return r.length == 0 })) {
-      baseView = false;
-      readView = true;
-    }
     toggleVisibility(unsupportedMessage, summaryView || coverageView);
     toggleVisibility(readOutlines, readView);
     toggleVisibility(variantOutlines, readView);
     toggleVisibility(readLetters, baseView);
     toggleVisibility(variantLetters, baseView);
     toggleVisibility(positionIndicator, baseView);
-
-    var sequenceStart = parseInt(x.domain()[0]);
-    var sequenceEnd = parseInt(x.domain()[1]);
-
     // TODO: Bring back coverage and summary views
+
     if (readView) {
-      // Read outlines
       readOutlines.attr("points", readOutlinePoints);
-
-      // Variant outlines
       variantOutlines
         .attr("x1", function(data) { return x(data.rx) + textWidth; })
         .attr("x2", function(data) { return x(data.rx) + textWidth; })
         .attr("y1", function(data) { return y(maxY - data.ry); })
         .attr("y2", function(data) { return y(maxY - data.ry) + textHeight; });
-
+      readLetters.remove();
+      
     } else if (baseView) {
-      // Read bases
-      readLetters.style('display', function(data, i) {
-            if (data.rx < sequenceStart || data.rx >= sequenceEnd - 1) {
-              return 'none';
-            } else {
-              return 'block';
-            }
-          })
-          .attr("x", function(data, i) {
+      readLetters = readLetters.data(function(read, i) { 
+        return read.readPieces.filter(function(letter) {
+          return letter.rx >= sequenceStart && letter.rx < sequenceEnd;
+        }); 
+      });
+
+      readLetters.enter().append('text')
+        .attr('class', 'letter')
+        .style('opacity', function(data, i) { return opacity(data.qual); })
+        .text(function(data, i) { return data.letter; });
+      readLetters.exit().remove();
+      
+      readLetters.attr("x", function(data, i) {
             return x(data.rx) + textWidth;
           })
           .attr("y", function(data, i) {
-            return y(data.ry) + textHeight/2;
+            return y(this.parentNode.__data__.yOrder) + textHeight/2;
           });
 
       // Red position highlight box
@@ -578,7 +607,7 @@ var readgraph = new function() {
       return '0,0';
     }
 
-    var startY = y(read.yOrder);
+    var startY = y(this.parentNode.__data__.yOrder);
     var endY = startY + barHeight;
     var midY = (startY + barHeight / 2);
 
@@ -654,14 +683,6 @@ var readgraph = new function() {
 
   // D3 object creation
 
-  var setYOrder = function(read, yOrder) {
-    read.yOrder = yOrder;
-
-    for (var r = 0; r < read.readPieces.length; r++) {
-      read.readPieces[r].ry = read.yOrder;
-    }
-  };
-
   var getGenotype = function(variant, call) {
 
     // TODO: Switch to genotype field when its ready
@@ -728,25 +749,33 @@ var readgraph = new function() {
     updateDisplay();
   };
 
-  var setReads = function(reads) {
-    var yTracks = [];
-    var readIds = {};
-    readStats = {};
+  var updateReads = function(reads) {
+    var newReadIds = {};
+    
     $.each(reads, function(readi, read) {
-      // Interpret the cigar
-      // TODO: Compare the read against a reference as well
-
       // TODO: Nobody is handing back actually unique ids right now
       read.id = (read.id || read.name) + read.position + read.cigar;
-      if (readIds[read.id]) {
+
+      if (newReadIds[read.id]) {
         showError('There is more than one read with the ID ' + read.id +
-            ' - this will cause display problems');
+            ' - extras ignored');
+        return;
       }
-      readIds[read.id] = true;
+      newReadIds[read.id] = true;
 
+      // Skip this read if we already have it.  Ideally the API would let us
+      // exclude the reads we already have.
+      if (read.id in readCache.readsById) {
+        if (('originalBases' in read) == ('originalBases' in readCache.readsById[read.id])) {        
+          return;
+        }
+        read.yOrder = readCache.readsById[read.id].yOrder;
+      } 
+      
+      // Interpret the cigar
+      // TODO: Compare the read against a reference as well
       read.name = read.name || read.id;
       read.readPieces = [];
-      read.index = readi;
       if (!read.cigar) {
         // Hack for unmapped reads
         read.length = 0;
@@ -818,58 +847,30 @@ var readgraph = new function() {
       // The 5th flag bit indicates this read is reversed
       read.reverse = (read.flags >> 4) % 2 == 1;
 
-      for (var i = 0; i < yTracks.length; i++) {
-        if (yTracks[i] < read.position) {
-          yTracks[i] = read.end;
-          setYOrder(read, i);
-          return;
+      // Create or update the entry in the cache, assuming we still want it.
+      // Note that we can't easily do this test above because we need to compute
+      // read.end first.
+      if (overlaps(read.position, read.end, readCache.desiredStart, readCache.desiredEnd)) {
+        // Find the lowest available track for this read.
+        // TODO: Use a more efficient algorithm.
+        trackUnavailable = [];
+        $.each(readCache.readsById, function(id, read2) {
+          if (overlaps(read.position, read.end, read2.position, read2.end)) {
+            trackUnavailable[read2.yOrder] = true;
+          }
+        });
+        for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
         }
+        readCache.readsById[read.id] = read;
       }
-
-      setYOrder(read, yTracks.length);
-      yTracks.push(read.end);
     });
-
-    readTrackLength = yTracks.length;
-
-    readGroup.selectAll('.read').remove();
-    if (reads.length == 0) {
-      // Update the data behind the graph
-      return;
-    }
-
-    var reads = readGroup.selectAll(".read").data(reads,
-        function(read){ return read.id; });
-
-    reads.enter().append("g")
-        .attr('class', 'read')
-        .attr('index', function(read, i) { return read.index; })
-        .on("mouseover", showRead)
-        .on("mouseout", deselectObject);
-
-    var outlines = reads.selectAll('.outline')
-        .data(function(read, i) { return [read];});
-    outlines.enter().append('polygon')
-        .attr('class', 'outline');
-
-    var baseView = getScaleLevel() > 5;
-    if (baseView) {
-      var letters = reads.selectAll(".letter")
-          .data(function(read, i) { return read.readPieces; });
-
-      letters.enter().append('text')
-          .attr('class', 'letter')
-          .style('opacity', function(data, i) { return opacity(data.qual); })
-          .text(function(data, i) { return data.letter; });
-    }
-    reads.exit().remove();
+    
     updateDisplay();
-  };
-
-
+  }
+    
   // Data loading
 
-  var makeQueryParams = function(sequenceStart, sequenceEnd, type) {
+  var makeQueryParams = function(sequenceStart, sequenceEnd, type, opt_bases) {
     var sets = _.where(setObjects, {type: type});
     if (sets.length == 0) {
       return null;
@@ -891,22 +892,50 @@ var readgraph = new function() {
     queryParams.sequenceEnd = parseInt(sequenceEnd);
     
     if (type == READSET_TYPE) {
-      var baseView = getScaleLevel() > 5;
-      var baseFields = baseView ? ',originalBases,baseQuality' : '';
+      var baseFields = opt_bases ? ',originalBases,baseQuality' : '';
       queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,cigar' + baseFields + ')';
     }
 
     return queryParams;
   };
 
-  var queryData = function(start, end) {
-    var readParams = makeQueryParams(start, end, READSET_TYPE);
+  var ensureReadsCached = function(start, end, bases) {
+    // Cache additional data than just what's requested so that we can do
+    // some small navigations quickly and without incurring redudant transfers.
+    // Smaller values of the cache factor result in more redundant read
+    // operations, larger values use more memory and reduce the likelihood of
+    // temporarily seeing missing reads.
+    // If the API allowed us to request (or exclude) specific reads, then we
+    // could be sure to query only for the reads we don't already have.
+    const readCacheFactor = 1;
+    var bufferSize = (end - start) * readCacheFactor;
+    
+    if (start - bufferSize >= readCache.desiredStart 
+        && end + bufferSize <= readCache.desiredEnd
+        && (readCache.desireBases || !bases)) {
+      return;
+    }
+    
+    readCache.desiredStart = start - 2 * bufferSize;
+    readCache.desiredEnd = end + 2 * bufferSize;
+    readCache.desireBases = bases;
+    // TODO: Don't requery the current region for the pan case.
+    queryData(readCache.desiredStart, readCache.desiredEnd, readCache.desireBases);
+    
+    // Discard any cached reads that are now entirely outside our desired window.
+    $.each(readCache.readsById, function(id, read) {
+      if (read.position > readCache.desiredEnd || read.end < readCache.desiredStart) {
+        delete readCache.readsById[id];
+      }
+    });
+  };
+  
+  var queryData = function(start, end, bases) {
+    var readParams = makeQueryParams(start, end, READSET_TYPE, bases);
     var variantParams = makeQueryParams(start, end, CALLSET_TYPE);
 
     if (readParams) {
-      callXhr('/api/reads', readParams, setReads);
-    } else {
-      setReads([]);
+      callXhr('/api/reads', readParams, updateReads);
     }
     if (variantParams) {
       callXhr('/api/variants', variantParams, setVariants);
@@ -915,6 +944,7 @@ var readgraph = new function() {
     }
   };
 
+  // TODO: Fix handling of overlapping loads.
   var showSpinner = function(show) {
     if (show == spinnerShown) {
       return;
@@ -938,8 +968,15 @@ var readgraph = new function() {
     showSpinner(true);
     $.getJSON(url, queryParams)
         .done(function(res) {
-          var data = (opt_data || []).concat(res.reads || res.variants || []);
-          handler(data);
+          var data;
+          if (res.reads) {
+            // Reads are handled incrementally, but variants aren't yet.
+            // Eventually variants will be updated to follow the same pattern.
+            handler(res.reads)
+          } else {
+            data = (opt_data || []).concat(res.variants || []);
+            handler(data);
+          }
 
           if (res.nextPageToken) {
             queryParams['pageToken'] = res.nextPageToken;

From b57a9ab6e058bd1ffbf030ae4eefcf19aba05ef3 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 11:11:26 -0400
Subject: [PATCH 26/42] Several improvements to read loading and navigation

When zooming into base view, temporarily show the read outlines while we wait for the
bases to load.  Also properly preserve track locations when shifting between read
and base view.

Fix spinner / load logging to properly handle overlapping loads.

Ensure handleZoomEnd (and hence JSON read requests) is done exactly once for each navigation.
Previously there was some issue where clicking on the zoom buttons would call this once with
a bogus position (clearing the read cache) then call it again with the correct position.

When querying a subset of fields, request 1024 results (the max) instead of 256
to avoid getting any pages that are each small in size.  Add total read volume
logging to the client.  Tweak the cache factors to reduce the loads / delays
when at largest scale.
---
 main.py                |   2 +-
 static/js/readgraph.js | 114 +++++++++++++++++++++++++----------------
 2 files changed, 70 insertions(+), 46 deletions(-)

diff --git a/main.py b/main.py
index 806af7e..f7ec489 100644
--- a/main.py
+++ b/main.py
@@ -189,7 +189,7 @@ def get(self):
     fields = self.request.get('fields')
     params = ''
     if fields:
-      params = 'fields=%s' % fields
+      params = 'fields=%s&maxResults=1024' % fields
     pageToken = self.request.get('pageToken')
     if pageToken:
       body['pageToken'] = pageToken
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 37f1f2e..0dad1cc 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -46,8 +46,6 @@ var readgraph = new function() {
   var readTrackLength = 0;
   var callsetTrackLength = 0;
 
-  var spinnerShown = false;
-
   var readCache = {
       desireBases: false,
       desiredStart: 0,
@@ -122,7 +120,6 @@ var readgraph = new function() {
     var newX = x(position);
     newX = zoom.translate()[0] - newX + width / 2;
     zoom.translate([newX, 0]);
-    handleZoomEnd();
   };
 
   var setupRun = false;
@@ -224,8 +221,6 @@ var readgraph = new function() {
       newZoom = Math.max(1, newZoom);
       newZoom = Math.min(maxZoom, newZoom);
       zoom.scale(newZoom);
-
-      handleZoomEnd();
       moveToSequencePosition(middleX);
     };
 
@@ -352,9 +347,9 @@ var readgraph = new function() {
     var zoomLevel = baseView ? maxZoom : maxZoom / zoomLevelChange; // Read level
     if (zoom.scale() != zoomLevel) {
       zoom.scale(zoomLevel);
-      handleZoom();
     }
     moveToSequencePosition(position);
+    handleZoomEnd();
   };
 
   var addImage = function(parent, name, width, height, x, y,
@@ -397,7 +392,7 @@ var readgraph = new function() {
     maxZoom = Math.ceil(Math.max(1, sequence['length'] / minRange));
     zoomLevelChange = Math.pow(maxZoom, 1/6);
     zoom.x(x).scaleExtent([1, maxZoom]).size([width, height]);
-
+    
     $('#jumpDiv').show();
   };
 
@@ -497,6 +492,14 @@ var readgraph = new function() {
     var variantOutlines = variants.selectAll(".outline");
     var variantLetters = variants.selectAll(".letter");
 
+    // If we are trying to do base view but have no reads with bases yet, then
+    // just show reads for now.
+    if (baseView && readsInView.length 
+        && readsInView.every(function (r) { return r.readPieces.length == 0; })) {
+      baseView = false;
+      readView = true;
+    }
+    
     toggleVisibility(unsupportedMessage, summaryView || coverageView);
     toggleVisibility(readOutlines, readView);
     toggleVisibility(variantOutlines, readView);
@@ -769,7 +772,6 @@ var readgraph = new function() {
         if (('originalBases' in read) == ('originalBases' in readCache.readsById[read.id])) {        
           return;
         }
-        read.yOrder = readCache.readsById[read.id].yOrder;
       } 
       
       // Interpret the cigar
@@ -850,16 +852,22 @@ var readgraph = new function() {
       // Create or update the entry in the cache, assuming we still want it.
       // Note that we can't easily do this test above because we need to compute
       // read.end first.
-      if (overlaps(read.position, read.end, readCache.desiredStart, readCache.desiredEnd)) {
-        // Find the lowest available track for this read.
-        // TODO: Use a more efficient algorithm.
-        trackUnavailable = [];
-        $.each(readCache.readsById, function(id, read2) {
-          if (overlaps(read.position, read.end, read2.position, read2.end)) {
-            trackUnavailable[read2.yOrder] = true;
+      if (overlaps(read.position, read.end, readCache.desiredStart, readCache.desiredEnd)
+              && ('originalBases' in read) == readCache.desireBases) {
+        var existingRead = readCache.readsById[read.id];
+        if (existingRead) {
+          read.yOrder = existingRead.yOrder;
+        } else {
+          // Find the lowest available track for this read.
+          // TODO: Use a more efficient algorithm.
+          trackUnavailable = [];
+          $.each(readCache.readsById, function(id, read2) {
+            if (overlaps(read.position, read.end, read2.position, read2.end)) {
+              trackUnavailable[read2.yOrder] = true;
+            }
+          });
+          for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
           }
-        });
-        for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
         }
         readCache.readsById[read.id] = read;
       }
@@ -906,18 +914,19 @@ var readgraph = new function() {
     // operations, larger values use more memory and reduce the likelihood of
     // temporarily seeing missing reads.
     // If the API allowed us to request (or exclude) specific reads, then we
-    // could be sure to query only for the reads we don't already have.
-    const readCacheFactor = 1;
-    var bufferSize = (end - start) * readCacheFactor;
-    
-    if (start - bufferSize >= readCache.desiredStart 
-        && end + bufferSize <= readCache.desiredEnd
+    // could be sure to query only for the reads we don't already have (perhaps
+    // we should emulate that in the python layer).
+    const minCacheFactor = 0.5;
+    const maxCacheFactor = 1;
+    var windowSize = end - start;    
+    if (start - windowSize * minCacheFactor >= readCache.desiredStart 
+        && end + windowSize * minCacheFactor <= readCache.desiredEnd
         && (readCache.desireBases || !bases)) {
       return;
     }
     
-    readCache.desiredStart = start - 2 * bufferSize;
-    readCache.desiredEnd = end + 2 * bufferSize;
+    readCache.desiredStart = start - windowSize * maxCacheFactor;
+    readCache.desiredEnd = end + windowSize * maxCacheFactor;
     readCache.desireBases = bases;
     // TODO: Don't requery the current region for the pan case.
     queryData(readCache.desiredStart, readCache.desiredEnd, readCache.desireBases);
@@ -944,30 +953,45 @@ var readgraph = new function() {
     }
   };
 
-  // TODO: Fix handling of overlapping loads.
-  var showSpinner = function(show) {
-    if (show == spinnerShown) {
-      return;
+  
+  var pendingLoads = 0;
+  var totalLoads = 0;
+  var startLoadMonitor = function() {
+    if (!pendingLoads) {
+      spinner.style('display', 'block');      
     }
-    spinnerShown = show;
-    spinner.style('display', show ? 'block' : 'none');
+    pendingLoads++;
+    var loadIndex = totalLoads++;
+    var timeLabel = 'readgraph load[' + loadIndex + ']'
     if ('time' in console) {
-      var timeLabel = 'readgraph loading';
-      if (show) {
-        console.time(timeLabel);
-      } else {
-        console.timeEnd(timeLabel);
-      }
+      console.time(timeLabel);
     }
     if ('timeStamp' in console) {
-      console.timeStamp('readgraph load ' + (show ? 'start' : 'finish'));
+      console.timeStamp(timeLabel + ' start');      
     }
+    
+    // Callers should invoke this callback on completion
+    return function() {
+      pendingLoads--;
+      if (!pendingLoads) {
+        spinner.style('display', 'none');      
+      }
+      if ('time' in console) {
+        console.timeEnd(timeLabel);
+      }
+      if ('timeStamp' in console) {
+        console.timeStamp(timeLabel + ' finish');      
+      }
+    };
   }
-
-  var callXhr = function(url, queryParams, handler, opt_data) {
-    showSpinner(true);
+  
+  var totalReadBytes = 0;
+  var callXhr = function(url, queryParams, handler, opt_monitor, opt_data) {
+    var onComplete = opt_monitor || startLoadMonitor();
     $.getJSON(url, queryParams)
-        .done(function(res) {
+        .done(function(res, status, jqXHR) {
+          totalReadBytes += Number.parseInt(jqXHR.getResponseHeader('Content-Length'));
+          console.log('readgraph load total: ' + Math.round(totalReadBytes/1024) + 'kb');
           var data;
           if (res.reads) {
             // Reads are handled incrementally, but variants aren't yet.
@@ -980,13 +1004,13 @@ var readgraph = new function() {
 
           if (res.nextPageToken) {
             queryParams['pageToken'] = res.nextPageToken;
-            callXhr(url, queryParams, handler, data);
+            callXhr(url, queryParams, handler, onComplete, data);
           } else {
-            showSpinner(false);
+            onComplete();
           }
         })
         .fail(function() {
-          showSpinner(false);
+          onComplete();
         });
   };
 

From e3929ae598309e2b3033a52177c3788e6e499942 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 11:24:35 -0400
Subject: [PATCH 27/42] Fix maxResults behavior and extend client read volume
 logging

---
 main.py                | 3 ++-
 static/js/readgraph.js | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/main.py b/main.py
index f7ec489..300a26a 100644
--- a/main.py
+++ b/main.py
@@ -189,7 +189,8 @@ def get(self):
     fields = self.request.get('fields')
     params = ''
     if fields:
-      params = 'fields=%s&maxResults=1024' % fields
+      params = 'fields=%s' % fields
+      body['maxResults'] = 1024
     pageToken = self.request.get('pageToken')
     if pageToken:
       body['pageToken'] = pageToken
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 0dad1cc..d4b9cc3 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -990,10 +990,11 @@ var readgraph = new function() {
     var onComplete = opt_monitor || startLoadMonitor();
     $.getJSON(url, queryParams)
         .done(function(res, status, jqXHR) {
-          totalReadBytes += Number.parseInt(jqXHR.getResponseHeader('Content-Length'));
-          console.log('readgraph load total: ' + Math.round(totalReadBytes/1024) + 'kb');
           var data;
           if (res.reads) {
+            var len = Number.parseInt(jqXHR.getResponseHeader('Content-Length'));
+            totalReadBytes += len;
+            console.log('readgraph ' + res.reads.length + ' reads (' + Math.round(len/1024) + 'kb), total ' + Math.round(totalReadBytes/1024) + 'kb');
             // Reads are handled incrementally, but variants aren't yet.
             // Eventually variants will be updated to follow the same pattern.
             handler(res.reads)

From 11b8d862a5cc42d44f55489bca7b64d2a01803d7 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 11:50:45 -0400
Subject: [PATCH 28/42] Properly set the content type in the response

---
 main.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/main.py b/main.py
index 300a26a..75862fd 100644
--- a/main.py
+++ b/main.py
@@ -127,6 +127,7 @@ def get_content(self, path, method='POST', body=None, params=''):
     return content
 
   def write_content(self, path, method='POST', body=None, params=''):
+    self.response.headers['Content-Type'] = "application/json"
     self.response.write(json.dumps(self.get_content(path, method, body, params)))
 
 

From 534d47e67b1379166976c0d971a1af9557d1a97e Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 16:42:19 -0400
Subject: [PATCH 29/42] Add a .gitignore file to list files that likely
 shouldn't be included in the repo (like google_api_key.txt).

---
 .gitignore | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7b713f6..bf2f680 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,7 @@
-*~
+.gitignore
+.project
+.pydevproject
+.settings
+google_api_key.txt
+static/js/d3.js
 *.pyc
-.idea
\ No newline at end of file

From c6945a4ce36bff446f8a49f061fa23e20cac9599 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 16:42:56 -0400
Subject: [PATCH 30/42] Read search optimization: don't request reads in the
 range we already have.

This substantially reduces transfer volume (and update latency) in some cases,
like panning along at full read-view scale.

Also remove trailing whitespace I accidentally added.
---
 static/js/main.js      |   2 +-
 static/js/readgraph.js | 110 ++++++++++++++++++++++++++---------------
 2 files changed, 70 insertions(+), 42 deletions(-)

diff --git a/static/js/main.js b/static/js/main.js
index e2bba4a..e8b89b5 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -56,7 +56,7 @@ function loadSet(backend, readsetIds, callsetIds, opt_location, setType, id) {
   }
 
   showMessage('Loading data');
-  
+
   $.getJSON('/api/sets', {backend: backend, setType: setType, setId: id})
     .done(function(res) {
       var sequenceData = res.contigs || res.fileData[0].refSequences;
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index d4b9cc3..11ac731 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -52,7 +52,7 @@ var readgraph = new function() {
       desiredEnd: 0,
       readsById: {}
   };
-  
+
   // Dom elements
   var svg, axisGroup, readGroup, readDiv, variantDiv, spinner = null;
   var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
@@ -79,10 +79,10 @@ var readgraph = new function() {
   var getScaleLevel = function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
   };
-  
+
   // Return whether two sequences overlap at all
   var overlaps = function(start1, end1, start2, end2) {
-    return start1 < end2 && end1 > start2; 
+    return start1 < end2 && end1 > start2;
   }
 
   // Called at high frequency for any navigation of the UI (not just zooming).
@@ -99,7 +99,7 @@ var readgraph = new function() {
     d3.select('.zoomLevel').attr('y', (6 - getScaleLevel()) * 24 + 38);
     updateDisplay();
   };
-  
+
   // Called when a navigation operation has completed.  This is the place to
   // do more expensive operations than simple UI updates.  Should not be
   // called frequently.
@@ -111,7 +111,7 @@ var readgraph = new function() {
       var sequenceEnd = parseInt(x.domain()[1]);
       ensureReadsCached(sequenceStart, sequenceEnd, scaleLevel > 5);
     }
-  } 
+  }
 
   var moveToSequencePosition = function(position) {
     position = Math.max(0, position);
@@ -392,7 +392,7 @@ var readgraph = new function() {
     maxZoom = Math.ceil(Math.max(1, sequence['length'] / minRange));
     zoomLevelChange = Math.pow(maxZoom, 1/6);
     zoom.x(x).scaleExtent([1, maxZoom]).size([width, height]);
-    
+
     $('#jumpDiv').show();
   };
 
@@ -458,7 +458,7 @@ var readgraph = new function() {
   // displayed (eg. offscreen letters).
   var updateDisplay = function() {
     var maxY = updateHeight();
-    
+
     var scaleLevel = getScaleLevel();
     var summaryView = scaleLevel < 2;
     var coverageView = scaleLevel == 2 || scaleLevel == 3;
@@ -486,7 +486,7 @@ var readgraph = new function() {
     readOutlines.enter().append('polygon')
       .attr('class', 'outline');
     readOutlines.exit().remove();
-    
+
     var readLetters = reads.selectAll(".letter");
 
     var variantOutlines = variants.selectAll(".outline");
@@ -494,12 +494,12 @@ var readgraph = new function() {
 
     // If we are trying to do base view but have no reads with bases yet, then
     // just show reads for now.
-    if (baseView && readsInView.length 
+    if (baseView && readsInView.length
         && readsInView.every(function (r) { return r.readPieces.length == 0; })) {
       baseView = false;
       readView = true;
     }
-    
+
     toggleVisibility(unsupportedMessage, summaryView || coverageView);
     toggleVisibility(readOutlines, readView);
     toggleVisibility(variantOutlines, readView);
@@ -516,12 +516,12 @@ var readgraph = new function() {
         .attr("y1", function(data) { return y(maxY - data.ry); })
         .attr("y2", function(data) { return y(maxY - data.ry) + textHeight; });
       readLetters.remove();
-      
+
     } else if (baseView) {
-      readLetters = readLetters.data(function(read, i) { 
+      readLetters = readLetters.data(function(read, i) {
         return read.readPieces.filter(function(letter) {
           return letter.rx >= sequenceStart && letter.rx < sequenceEnd;
-        }); 
+        });
       });
 
       readLetters.enter().append('text')
@@ -529,7 +529,7 @@ var readgraph = new function() {
         .style('opacity', function(data, i) { return opacity(data.qual); })
         .text(function(data, i) { return data.letter; });
       readLetters.exit().remove();
-      
+
       readLetters.attr("x", function(data, i) {
             return x(data.rx) + textWidth;
           })
@@ -754,7 +754,7 @@ var readgraph = new function() {
 
   var updateReads = function(reads) {
     var newReadIds = {};
-    
+
     $.each(reads, function(readi, read) {
       // TODO: Nobody is handing back actually unique ids right now
       read.id = (read.id || read.name) + read.position + read.cigar;
@@ -769,11 +769,11 @@ var readgraph = new function() {
       // Skip this read if we already have it.  Ideally the API would let us
       // exclude the reads we already have.
       if (read.id in readCache.readsById) {
-        if (('originalBases' in read) == ('originalBases' in readCache.readsById[read.id])) {        
+        if (('originalBases' in read) == ('originalBases' in readCache.readsById[read.id])) {
           return;
         }
-      } 
-      
+      }
+
       // Interpret the cigar
       // TODO: Compare the read against a reference as well
       read.name = read.name || read.id;
@@ -872,10 +872,10 @@ var readgraph = new function() {
         readCache.readsById[read.id] = read;
       }
     });
-    
+
     updateDisplay();
   }
-    
+
   // Data loading
 
   var makeQueryParams = function(sequenceStart, sequenceEnd, type, opt_bases) {
@@ -898,7 +898,7 @@ var readgraph = new function() {
     queryParams.sequenceName = currentSequence.name;
     queryParams.sequenceStart = parseInt(sequenceStart);
     queryParams.sequenceEnd = parseInt(sequenceEnd);
-    
+
     if (type == READSET_TYPE) {
       var baseFields = opt_bases ? ',originalBases,baseQuality' : '';
       queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,cigar' + baseFields + ')';
@@ -908,29 +908,57 @@ var readgraph = new function() {
   };
 
   var ensureReadsCached = function(start, end, bases) {
+
+    // Are we switching between base and read view?
+    var addBases = false;
+    if (bases != readCache.desireBases) {
+      if (bases) {
+        addBases = true;
+      } else {
+        // clear any stored bases to avoid potentially holding onto a lot of
+        // extra memory indefinitely
+        $.each(readCache.readsById, function(id, read) {
+          read.readPieces = [];
+        });
+      }
+    }
+
     // Cache additional data than just what's requested so that we can do
     // some small navigations quickly and without incurring redudant transfers.
     // Smaller values of the cache factor result in more redundant read
     // operations, larger values use more memory and reduce the likelihood of
     // temporarily seeing missing reads.
-    // If the API allowed us to request (or exclude) specific reads, then we
-    // could be sure to query only for the reads we don't already have (perhaps
-    // we should emulate that in the python layer).
     const minCacheFactor = 0.5;
     const maxCacheFactor = 1;
-    var windowSize = end - start;    
-    if (start - windowSize * minCacheFactor >= readCache.desiredStart 
+    var windowSize = end - start;
+    if (start - windowSize * minCacheFactor >= readCache.desiredStart
         && end + windowSize * minCacheFactor <= readCache.desiredEnd
-        && (readCache.desireBases || !bases)) {
+        && !addBases) {
       return;
     }
-    
-    readCache.desiredStart = start - windowSize * maxCacheFactor;
-    readCache.desiredEnd = end + windowSize * maxCacheFactor;
+
+
+    var desiredStart = start - windowSize * maxCacheFactor;
+    var desiredEnd = end + windowSize * maxCacheFactor;
+
+    if (overlaps(desiredStart, desiredEnd, readCache.desiredStart, readCache.desiredEnd)
+        && !addBases) {
+      // Don't re-request the reads we already have.  This will still retransfer
+      // reads which overlap the boundaries, but that should be small in practice.
+      if (desiredStart < readCache.desiredStart) {
+        queryData(desiredStart, readCache.desiredStart, bases);
+      }
+      if (desiredEnd > readCache.desiredEnd) {
+        queryData(readCache.desiredEnd, desiredEnd, bases);
+      }
+    } else {
+      queryData(desiredStart, desiredEnd, bases);
+    }
+
+    readCache.desiredStart = desiredStart;
+    readCache.desiredEnd = desiredEnd;
     readCache.desireBases = bases;
-    // TODO: Don't requery the current region for the pan case.
-    queryData(readCache.desiredStart, readCache.desiredEnd, readCache.desireBases);
-    
+
     // Discard any cached reads that are now entirely outside our desired window.
     $.each(readCache.readsById, function(id, read) {
       if (read.position > readCache.desiredEnd || read.end < readCache.desiredStart) {
@@ -938,7 +966,7 @@ var readgraph = new function() {
       }
     });
   };
-  
+
   var queryData = function(start, end, bases) {
     var readParams = makeQueryParams(start, end, READSET_TYPE, bases);
     var variantParams = makeQueryParams(start, end, CALLSET_TYPE);
@@ -953,12 +981,12 @@ var readgraph = new function() {
     }
   };
 
-  
+
   var pendingLoads = 0;
   var totalLoads = 0;
   var startLoadMonitor = function() {
     if (!pendingLoads) {
-      spinner.style('display', 'block');      
+      spinner.style('display', 'block');
     }
     pendingLoads++;
     var loadIndex = totalLoads++;
@@ -967,24 +995,24 @@ var readgraph = new function() {
       console.time(timeLabel);
     }
     if ('timeStamp' in console) {
-      console.timeStamp(timeLabel + ' start');      
+      console.timeStamp(timeLabel + ' start');
     }
-    
+
     // Callers should invoke this callback on completion
     return function() {
       pendingLoads--;
       if (!pendingLoads) {
-        spinner.style('display', 'none');      
+        spinner.style('display', 'none');
       }
       if ('time' in console) {
         console.timeEnd(timeLabel);
       }
       if ('timeStamp' in console) {
-        console.timeStamp(timeLabel + ' finish');      
+        console.timeStamp(timeLabel + ' finish');
       }
     };
   }
-  
+
   var totalReadBytes = 0;
   var callXhr = function(url, queryParams, handler, opt_monitor, opt_data) {
     var onComplete = opt_monitor || startLoadMonitor();

From 8b3a7f4ab74e013986a7a46f0a80b0271614b62d Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Wed, 9 Jul 2014 18:00:17 -0400
Subject: [PATCH 31/42] Fix EBI backend support by emulating partial response
 support for reads

NCBI and Google support partial responses (https://developers.google.com/genomics/performance#partial-response),
but EBI does not.  Things are simpler and more performant if the client
can assume this support exists.  So emulate it for EBI in python.
---
 main.py                | 35 ++++++++++++++++++++++++++++-------
 static/js/readgraph.js |  6 ++++--
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/main.py b/main.py
index 75862fd..68f592e 100644
--- a/main.py
+++ b/main.py
@@ -41,12 +41,14 @@
 SUPPORTED_BACKENDS = {
   'NCBI' : {'name': 'NCBI',
             'url': 'http://trace.ncbi.nlm.nih.gov/Traces/gg/%s?%s',
+            'supportsPartialResponse': True,
             'datasets': {'SRP034507': 'SRP034507', 'SRP029392': 'SRP029392'}},
   'EBI' : {'name': 'EBI',
             'url': 'http://193.62.52.16/%s?%s',
             'datasets': {'All data': 'data'}},
   'LOCAL' : {'name': 'Local',
              'url': 'http://localhost:5000/%s?%s',
+             'supportsPartialResponse': True,
              'datasets': {'All': ''}},
 }
 
@@ -61,6 +63,7 @@
       'url': 'https://www.googleapis.com/genomics/v1beta/%s?key='
              + api_key + '&%s',
       'supportsNameFilter': True,
+      'supportsPartialResponse': True,
       #'supportsCallsets': True,
       'datasets': {'1000 Genomes': '376902546192',
                    'DREAM SMC Challenge': '337315832689',
@@ -96,6 +99,9 @@ def get_backend(self):
   def supports_name_filter(self):
     return SUPPORTED_BACKENDS[self.get_backend()].has_key('supportsNameFilter')
 
+  def supports_partial_response(self):
+    return SUPPORTED_BACKENDS[self.get_backend()].has_key('supportsPartialResponse')
+
   def get_base_api_url(self):
     return SUPPORTED_BACKENDS[self.get_backend()]['url']
 
@@ -126,9 +132,12 @@ def get_content(self, path, method='POST', body=None, params=''):
     logging.info('get_content {}: {}kb {}s'.format(uri, contentLen/1024, time.clock() - startTime))
     return content
 
-  def write_content(self, path, method='POST', body=None, params=''):
+  def write_response(self, content):
     self.response.headers['Content-Type'] = "application/json"
-    self.response.write(json.dumps(self.get_content(path, method, body, params)))
+    self.response.write(json.dumps(content))
+    
+  def write_content(self, path, method='POST', body=None, params=''):
+    self.write_response(self.get_content(path, method, body, params))
 
 
 class SetSearchHandler(BaseRequestHandler):
@@ -187,15 +196,27 @@ def get(self):
       'sequenceStart': max(0, int(self.request.get('sequenceStart'))),
       'sequenceEnd': int(self.request.get('sequenceEnd')),
     }
-    fields = self.request.get('fields')
+    readFields = self.request.get('readFields')
     params = ''
-    if fields:
-      params = 'fields=%s' % fields
-      body['maxResults'] = 1024
+    if readFields and self.supports_partial_response():
+        params = 'fields=nextPageToken,reads(%s)' % readFields
+        body['maxResults'] = 1024
     pageToken = self.request.get('pageToken')
     if pageToken:
       body['pageToken'] = pageToken
-    self.write_content('reads/search', body=body, params=params)
+
+    content = self.get_content('reads/search', body=body, params=params)
+
+    # Emulate support for partial responses by supplying only the
+    # requested fields to the client.
+    if readFields and not self.supports_partial_response():
+      fields = readFields.split(',')
+      def filterKeys(dict, keys):
+        return { key: dict[key] for key in keys }
+      newReads = [ filterKeys(read, fields) for read in content['reads']]
+      content['reads'] = newReads
+      
+    self.write_response(content)
 
 
 class VariantSearchHandler(BaseRequestHandler):
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 11ac731..2619523 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -901,7 +901,7 @@ var readgraph = new function() {
 
     if (type == READSET_TYPE) {
       var baseFields = opt_bases ? ',originalBases,baseQuality' : '';
-      queryParams.fields = 'nextPageToken,reads(name,position,matePosition,mappingQuality,cigar' + baseFields + ')';
+      queryParams.readFields = 'name,position,matePosition,mappingQuality,cigar' + baseFields;
     }
 
     return queryParams;
@@ -1022,7 +1022,9 @@ var readgraph = new function() {
           if (res.reads) {
             var len = Number.parseInt(jqXHR.getResponseHeader('Content-Length'));
             totalReadBytes += len;
-            console.log('readgraph ' + res.reads.length + ' reads (' + Math.round(len/1024) + 'kb), total ' + Math.round(totalReadBytes/1024) + 'kb');
+            console.log('readgraph ' + res.reads.length
+              + (res.reads.length && 'originalBases' in res.reads[0] ? ' full' : ' partial')
+              + ' reads (' + Math.round(len/1024) + 'kb), total ' + Math.round(totalReadBytes/1024) + 'kb');
             // Reads are handled incrementally, but variants aren't yet.
             // Eventually variants will be updated to follow the same pattern.
             handler(res.reads)

From 33e852c8f3fd51d44896d325db27878b720a3ae0 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 11:57:27 -0400
Subject: [PATCH 32/42] Refactor readCache into it's own class

I'm about to make it more complicated, it should be abstracted.

Also clear redundant fields like originalBases after they're processed to reduce memory usage.

Add simple assert mechanism to help catch logic errors.
---
 main.html              |   1 +
 static/js/main.js      |   6 +++
 static/js/readgraph.js | 105 +++++++++++------------------------------
 3 files changed, 35 insertions(+), 77 deletions(-)

diff --git a/main.html b/main.html
index a57f9d7..314e873 100644
--- a/main.html
+++ b/main.html
@@ -28,6 +28,7 @@
   <link rel="stylesheet" href="/https/github.com/static/css/main.css">
   <script src="/https/github.com/static/js/main.js" charset="utf-8"></script>
   <script src="/https/github.com/static/js/readgraph.js" charset="utf-8"></script>
+  <script src="/https/github.com/static/js/readcache.js" charset="utf-8"></script>
 </head>
 
 <body>
diff --git a/static/js/main.js b/static/js/main.js
index e8b89b5..5493aa7 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -45,6 +45,12 @@ function showAlert(message, type) {
   }, type == 'danger' ? 5000 : 3000);
 }
 
+function assert(condition) {
+  if (!condition) {
+    console.error('assert failed');
+  }
+}
+
 function closeButton() {
   return $('<button type="button" class="close" aria-hidden="true">&times;</button>');
 }
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 2619523..bf55b98 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -46,13 +46,6 @@ var readgraph = new function() {
   var readTrackLength = 0;
   var callsetTrackLength = 0;
 
-  var readCache = {
-      desireBases: false,
-      desiredStart: 0,
-      desiredEnd: 0,
-      readsById: {}
-  };
-
   // Dom elements
   var svg, axisGroup, readGroup, readDiv, variantDiv, spinner = null;
   var hoverline, positionIndicator, positionIndicatorBg, positionIndicatorText;
@@ -80,11 +73,6 @@ var readgraph = new function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
   };
 
-  // Return whether two sequences overlap at all
-  var overlaps = function(start1, end1, start2, end2) {
-    return start1 < end2 && end1 > start2;
-  }
-
   // Called at high frequency for any navigation of the UI (not just zooming).
   // Should only do low-latency operations.
   var handleZoom = function() {
@@ -468,7 +456,7 @@ var readgraph = new function() {
     var sequenceStart = parseInt(x.domain()[0]);
     var sequenceEnd = parseInt(x.domain()[1]);
 
-    var readsInView = d3.values(readCache.readsById).filter(function(read) {
+    var readsInView = readCache.getReads().filter(function(read) {
       return overlaps(read.position, read.end, sequenceStart, sequenceEnd);
     });
     var reads = readGroup.selectAll(".read").data(readsInView,
@@ -766,18 +754,16 @@ var readgraph = new function() {
       }
       newReadIds[read.id] = true;
 
-      // Skip this read if we already have it.  Ideally the API would let us
-      // exclude the reads we already have.
-      if (read.id in readCache.readsById) {
-        if (('originalBases' in read) == ('originalBases' in readCache.readsById[read.id])) {
-          return;
-        }
+      // Skip this read if we already have exactly it.
+      if (readCache.hasRead(read)) {
+        return;
       }
 
       // Interpret the cigar
       // TODO: Compare the read against a reference as well
       read.name = read.name || read.id;
       read.readPieces = [];
+
       if (!read.cigar) {
         // Hack for unmapped reads
         read.length = 0;
@@ -797,7 +783,7 @@ var readgraph = new function() {
         });
       };
 
-      var bases;
+      var bases = null;
       if (read.originalBases) {
         bases = read.originalBases.split('');
       }
@@ -845,36 +831,24 @@ var readgraph = new function() {
         }
       }
 
+      // Remove data that's now redundant with readPieces.
+      if (bases) {
+        delete read.originalBases;
+        delete read.baseQuality;
+      }
+
       read.end = read.position + read.length;
       // The 5th flag bit indicates this read is reversed
       read.reverse = (read.flags >> 4) % 2 == 1;
 
       // Create or update the entry in the cache, assuming we still want it.
-      // Note that we can't easily do this test above because we need to compute
-      // read.end first.
-      if (overlaps(read.position, read.end, readCache.desiredStart, readCache.desiredEnd)
-              && ('originalBases' in read) == readCache.desireBases) {
-        var existingRead = readCache.readsById[read.id];
-        if (existingRead) {
-          read.yOrder = existingRead.yOrder;
-        } else {
-          // Find the lowest available track for this read.
-          // TODO: Use a more efficient algorithm.
-          trackUnavailable = [];
-          $.each(readCache.readsById, function(id, read2) {
-            if (overlaps(read.position, read.end, read2.position, read2.end)) {
-              trackUnavailable[read2.yOrder] = true;
-            }
-          });
-          for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
-          }
-        }
-        readCache.readsById[read.id] = read;
-      }
+      // Note that we can't easily test to see if we still want the read before
+      // doing the work above because we need to compute read.end.
+      readCache.addOrUpdateRead(read);
     });
 
     updateDisplay();
-  }
+  };
 
   // Data loading
 
@@ -909,20 +883,6 @@ var readgraph = new function() {
 
   var ensureReadsCached = function(start, end, bases) {
 
-    // Are we switching between base and read view?
-    var addBases = false;
-    if (bases != readCache.desireBases) {
-      if (bases) {
-        addBases = true;
-      } else {
-        // clear any stored bases to avoid potentially holding onto a lot of
-        // extra memory indefinitely
-        $.each(readCache.readsById, function(id, read) {
-          read.readPieces = [];
-        });
-      }
-    }
-
     // Cache additional data than just what's requested so that we can do
     // some small navigations quickly and without incurring redudant transfers.
     // Smaller values of the cache factor result in more redundant read
@@ -930,41 +890,32 @@ var readgraph = new function() {
     // temporarily seeing missing reads.
     const minCacheFactor = 0.5;
     const maxCacheFactor = 1;
+    var addingBases = bases && !readCache.wantBases;
     var windowSize = end - start;
-    if (start - windowSize * minCacheFactor >= readCache.desiredStart
-        && end + windowSize * minCacheFactor <= readCache.desiredEnd
-        && !addBases) {
+    if (start - windowSize * minCacheFactor >= readCache.start
+        && end + windowSize * minCacheFactor <= readCache.end
+        && !addingBases) {
       return;
     }
 
-
     var desiredStart = start - windowSize * maxCacheFactor;
     var desiredEnd = end + windowSize * maxCacheFactor;
 
-    if (overlaps(desiredStart, desiredEnd, readCache.desiredStart, readCache.desiredEnd)
-        && !addBases) {
+    if (overlaps(desiredStart, desiredEnd, readCache.start, readCache.end)
+        && !addingBases) {
       // Don't re-request the reads we already have.  This will still retransfer
       // reads which overlap the boundaries, but that should be small in practice.
-      if (desiredStart < readCache.desiredStart) {
-        queryData(desiredStart, readCache.desiredStart, bases);
+      if (desiredStart < readCache.start) {
+        queryData(desiredStart, readCache.start, bases);
       }
-      if (desiredEnd > readCache.desiredEnd) {
-        queryData(readCache.desiredEnd, desiredEnd, bases);
+      if (desiredEnd > readCache.end) {
+        queryData(readCache.end, desiredEnd, bases);
       }
     } else {
       queryData(desiredStart, desiredEnd, bases);
     }
 
-    readCache.desiredStart = desiredStart;
-    readCache.desiredEnd = desiredEnd;
-    readCache.desireBases = bases;
-
-    // Discard any cached reads that are now entirely outside our desired window.
-    $.each(readCache.readsById, function(id, read) {
-      if (read.position > readCache.desiredEnd || read.end < readCache.desiredStart) {
-        delete readCache.readsById[id];
-      }
-    });
+    readCache.setRange(desiredStart, desiredEnd, bases);
   };
 
   var queryData = function(start, end, bases) {
@@ -1027,7 +978,7 @@ var readgraph = new function() {
               + ' reads (' + Math.round(len/1024) + 'kb), total ' + Math.round(totalReadBytes/1024) + 'kb');
             // Reads are handled incrementally, but variants aren't yet.
             // Eventually variants will be updated to follow the same pattern.
-            handler(res.reads)
+            handler(res.reads);
           } else {
             data = (opt_data || []).concat(res.variants || []);
             handler(data);

From 18661d5cf07b6bc1659285213192de8dcdbb9215 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 12:05:40 -0400
Subject: [PATCH 33/42] Use JS strict mode for better error catching

---
 static/js/main.js      |  1 +
 static/js/readgraph.js | 14 ++++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/static/js/main.js b/static/js/main.js
index 5493aa7..279c50b 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+"use strict";
 
 var CALLSET_TYPE = "CALLSET";
 var READSET_TYPE = "READSET";
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index bf55b98..087c4c8 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+"use strict";
 
 var readgraph = new function() {
   var cigarMatcher = /([0-9]+[MIDNSHP=X])/gi
@@ -881,6 +882,9 @@ var readgraph = new function() {
     return queryParams;
   };
 
+  var MIN_CACHE_FACTOR = 0.5;
+  var MAX_CACHE_FACTOR = 1;
+
   var ensureReadsCached = function(start, end, bases) {
 
     // Cache additional data than just what's requested so that we can do
@@ -888,18 +892,16 @@ var readgraph = new function() {
     // Smaller values of the cache factor result in more redundant read
     // operations, larger values use more memory and reduce the likelihood of
     // temporarily seeing missing reads.
-    const minCacheFactor = 0.5;
-    const maxCacheFactor = 1;
     var addingBases = bases && !readCache.wantBases;
     var windowSize = end - start;
-    if (start - windowSize * minCacheFactor >= readCache.start
-        && end + windowSize * minCacheFactor <= readCache.end
+    if (start - windowSize * MIN_CACHE_FACTOR >= readCache.start
+        && end + windowSize * MIN_CACHE_FACTOR <= readCache.end
         && !addingBases) {
       return;
     }
 
-    var desiredStart = start - windowSize * maxCacheFactor;
-    var desiredEnd = end + windowSize * maxCacheFactor;
+    var desiredStart = start - windowSize * MAX_CACHE_FACTOR;
+    var desiredEnd = end + windowSize * MAX_CACHE_FACTOR;
 
     if (overlaps(desiredStart, desiredEnd, readCache.start, readCache.end)
         && !addingBases) {

From 2ea899f6b66d791915765d6cddf817d2694a5ac8 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 12:07:52 -0400
Subject: [PATCH 34/42] Add missing readcache.js file, and explicitly clear
 cache on sequence change.

Forgot to add this in the previous refactoring commit.

We were also potentially hanging onto cache data on sequence change (which
would only be a problem if the position range happened to overlap).
---
 app.yaml               |   2 +-
 static/js/readcache.js | 139 +++++++++++++++++++++++++++++++++++++++++
 static/js/readgraph.js |   1 +
 3 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 static/js/readcache.js

diff --git a/app.yaml b/app.yaml
index b1ca1c4..70ab3ef 100644
--- a/app.yaml
+++ b/app.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-application: gabrowse
+application: gabrowse-staging
 version: 1
 runtime: python27
 api_version: 1
diff --git a/static/js/readcache.js b/static/js/readcache.js
new file mode 100644
index 0000000..32f5f89
--- /dev/null
+++ b/static/js/readcache.js
@@ -0,0 +1,139 @@
+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/*
+ * Return whether two ranges overlap at all.
+ */
+function overlaps(start1, end1, start2, end2) {
+  return start1 < end2 && end1 > start2;
+};
+
+/*
+ * A data structure for keeping track of all the reads we have loaded.
+ * The objects in the cache have already been processed, and so must have
+ * their computed fields like end and readPieces (rather than the raw fields
+ * like originalBases).
+ */
+var readCache = new function() {
+  /*
+   * Whether we're caching base data.  All reads stored into the cache
+   * should have the 'originalBases' property if and only if this is true.
+   * When this changes from true to false, all cached base data is cleared.
+   * But when this changes from false to true, reads without base data are
+   * preserved until they can be updated to include the base data.
+   */
+  var wantBases = false;
+  this.__defineGetter__("wantBases", function() { return wantBases; });
+
+  /*
+   * The range [start,end) of sequence positions for which we are caching reads.
+   * There should never be any reads inside the cache that lie entirely outside
+   * this range.
+   */
+  var start = 0;
+  var end = 0;
+  this.__defineGetter__("start", function() { return start; });
+  this.__defineGetter__("end", function() { return end; });
+
+  // A map from read ID to read object.
+  var readsById = {};
+
+  /*
+   * Returns all the reads in the cache.
+   */
+  this.getReads = function() {
+    return d3.values(readsById);
+  };
+
+  /*
+   * Clear the entire cache.
+   */
+  this.clear = function() {
+    wantBases = false;
+    start = 0;
+    end = 0;
+    readsById = {};
+  };
+
+  /*
+   * Reset the cache range, clearing elements / base data that are no longer
+   * necessary.
+   */
+  this.setRange = function(newStart, newEnd, newBases) {
+    // Discard any cached reads that are now entirely outside our desired window.
+    $.each(readsById, function(id, read) {
+      if (!overlaps(read.position, read.end, newStart, newEnd)) {
+        delete readsById[id];
+      } else if (!newBases) {
+        read.readPieces = [];
+      }
+    });
+
+    start = newStart;
+    end = newEnd;
+    wantBases = newBases;
+  };
+
+  /*
+   * Return whether the cache already contains a read with the specified
+   * id and base status.
+   */
+  this.hasRead = function(id, bases) {
+    var existingRead = readsById[id];
+    if (existingRead && bases == ('readPieces' in existingRead)) {
+      return true;
+    }
+    return false;
+  };
+
+  /*
+   * Adds the supplied read to the cache if it's still relevant, assigning a
+   * free yOrder property to thr read.
+   * If a read with this ID already exists, updates the read (eg. to add
+   * or remove base data) without changing the yOrder.
+   */
+  this.addOrUpdateRead = function(read) {
+    // The read should have already been processed.
+    assert('end' in read);
+    assert('readPieces' in read);
+    assert(!('originalBases' in read));
+
+    // If we don't actually want this read anymore, do nothing.
+    if (!overlaps(read.position, read.end, start, end)
+            || (read.readPieces.length > 0) != wantBases) {
+      return;
+    }
+
+    var existingRead = readsById[read.id];
+    if (existingRead) {
+      read.yOrder = existingRead.yOrder;
+    } else {
+      // Find the lowest available track for this read.
+      // TODO: Use a more efficient algorithm.
+      var trackUnavailable = [];
+      $.each(readsById, function(id, read2) {
+        if (overlaps(read.position, read.end, read2.position, read2.end)) {
+          trackUnavailable[read2.yOrder] = true;
+        }
+      });
+      for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
+      }
+    }
+    readsById[read.id] = read;
+  };
+};
+
diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 087c4c8..bda3219 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -359,6 +359,7 @@ var readgraph = new function() {
   };
 
   var selectSequence = function(sequence) {
+    readCache.clear();
     currentSequence = sequence;
     $('.sequence').removeClass('active');
     var div = $('#' + sequenceId(sequence.name)).addClass('active');

From 07c0abcd380e3ec4fdc91467a887e22ed75cd004 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 13:18:05 -0400
Subject: [PATCH 35/42] Implement efficient yTrack management.

Since reads can now arrive in arbitrary orders, it's non-trivial to assign them
to tracks compactly.  The previous naive algorithm was O(n^2) in the number of
reads and at full read view was causing significant delays (sometimes several
seconds even on a fast desktop).

Fix this by maintaining a binary search tree for each track, so now updating
reads O(n log n) in the number of reads.

This adds a red black tree implementation from https://github.com/vadimg/js_bintrees
---
 main.html              |   1 +
 static/js/rbtree.js    | 493 +++++++++++++++++++++++++++++++++++++++++
 static/js/readcache.js |  74 ++++++-
 3 files changed, 561 insertions(+), 7 deletions(-)
 create mode 100644 static/js/rbtree.js

diff --git a/main.html b/main.html
index 314e873..3e300ca 100644
--- a/main.html
+++ b/main.html
@@ -23,6 +23,7 @@
   <script src="/https/github.com/static/js/d3.v3.min.js" charset="utf-8"></script>
   <script src="/https/github.com/static/js/underscore-min.js" charset="utf-8"></script>
   <script src="/https/github.com/static/js/jquery.bootpag.min.js" charset="utf-8"></script>
+  <script src="/https/github.com/static/js/rbtree.js" charset="utf-8"></script>
 
   <!-- Browser code -->
   <link rel="stylesheet" href="/https/github.com/static/css/main.css">
diff --git a/static/js/rbtree.js b/static/js/rbtree.js
new file mode 100644
index 0000000..b5bc328
--- /dev/null
+++ b/static/js/rbtree.js
@@ -0,0 +1,493 @@
+/* Copyright (C) 2011 by Vadim Graboys
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+RBTree = (function(window) {
+var global = window;
+var require = function(name) {
+    var fn = require.m[name];
+    if (fn.mod) {
+        return fn.mod.exports;
+    }
+
+    var mod = fn.mod = { exports: {} };
+    fn(mod, mod.exports);
+    return mod.exports;
+};
+
+require.m = {};
+require.m['./treebase'] = function(module, exports) {
+
+function TreeBase() {}
+
+// removes all nodes from the tree
+TreeBase.prototype.clear = function() {
+    this._root = null;
+    this.size = 0;
+};
+
+// returns node data if found, null otherwise
+TreeBase.prototype.find = function(data) {
+    var res = this._root;
+
+    while(res !== null) {
+        var c = this._comparator(data, res.data);
+        if(c === 0) {
+            return res.data;
+        }
+        else {
+            res = res.get_child(c > 0);
+        }
+    }
+
+    return null;
+};
+
+// returns iterator to node if found, null otherwise
+TreeBase.prototype.findIter = function(data) {
+    var res = this._root;
+    var iter = this.iterator();
+
+    while(res !== null) {
+        var c = this._comparator(data, res.data);
+        if(c === 0) {
+            iter._cursor = res;
+            return iter;
+        }
+        else {
+            iter._ancestors.push(res);
+            res = res.get_child(c > 0);
+        }
+    }
+
+    return null;
+};
+
+// Returns an interator to the tree node immediately before (or at) the element
+TreeBase.prototype.lowerBound = function(data) {
+    return this._bound(data, this._comparator);
+};
+
+// Returns an interator to the tree node immediately after (or at) the element
+TreeBase.prototype.upperBound = function(data) {
+    var cmp = this._comparator;
+
+    function reverse_cmp(a, b) {
+        return cmp(b, a);
+    }
+
+    return this._bound(data, reverse_cmp);
+};
+
+// returns null if tree is empty
+TreeBase.prototype.min = function() {
+    var res = this._root;
+    if(res === null) {
+        return null;
+    }
+
+    while(res.left !== null) {
+        res = res.left;
+    }
+
+    return res.data;
+};
+
+// returns null if tree is empty
+TreeBase.prototype.max = function() {
+    var res = this._root;
+    if(res === null) {
+        return null;
+    }
+
+    while(res.right !== null) {
+        res = res.right;
+    }
+
+    return res.data;
+};
+
+// returns a null iterator
+// call next() or prev() to point to an element
+TreeBase.prototype.iterator = function() {
+    return new Iterator(this);
+};
+
+// calls cb on each node's data, in order
+TreeBase.prototype.each = function(cb) {
+    var it=this.iterator(), data;
+    while((data = it.next()) !== null) {
+        cb(data);
+    }
+};
+
+// calls cb on each node's data, in reverse order
+TreeBase.prototype.reach = function(cb) {
+    var it=this.iterator(), data;
+    while((data = it.prev()) !== null) {
+        cb(data);
+    }
+};
+
+// used for lowerBound and upperBound
+TreeBase.prototype._bound = function(data, cmp) {
+    var cur = this._root;
+    var iter = this.iterator();
+
+    while(cur !== null) {
+        var c = this._comparator(data, cur.data);
+        if(c === 0) {
+            iter._cursor = cur;
+            return iter;
+        }
+        iter._ancestors.push(cur);
+        cur = cur.get_child(c > 0);
+    }
+
+    for(var i=iter._ancestors.length - 1; i >= 0; --i) {
+        cur = iter._ancestors[i];
+        if(cmp(data, cur.data) > 0) {
+            iter._cursor = cur;
+            iter._ancestors.length = i;
+            return iter;
+        }
+    }
+
+    iter._ancestors.length = 0;
+    return iter;
+};
+
+
+function Iterator(tree) {
+    this._tree = tree;
+    this._ancestors = [];
+    this._cursor = null;
+}
+
+Iterator.prototype.data = function() {
+    return this._cursor !== null ? this._cursor.data : null;
+};
+
+// if null-iterator, returns first node
+// otherwise, returns next node
+Iterator.prototype.next = function() {
+    if(this._cursor === null) {
+        var root = this._tree._root;
+        if(root !== null) {
+            this._minNode(root);
+        }
+    }
+    else {
+        if(this._cursor.right === null) {
+            // no greater node in subtree, go up to parent
+            // if coming from a right child, continue up the stack
+            var save;
+            do {
+                save = this._cursor;
+                if(this._ancestors.length) {
+                    this._cursor = this._ancestors.pop();
+                }
+                else {
+                    this._cursor = null;
+                    break;
+                }
+            } while(this._cursor.right === save);
+        }
+        else {
+            // get the next node from the subtree
+            this._ancestors.push(this._cursor);
+            this._minNode(this._cursor.right);
+        }
+    }
+    return this._cursor !== null ? this._cursor.data : null;
+};
+
+// if null-iterator, returns last node
+// otherwise, returns previous node
+Iterator.prototype.prev = function() {
+    if(this._cursor === null) {
+        var root = this._tree._root;
+        if(root !== null) {
+            this._maxNode(root);
+        }
+    }
+    else {
+        if(this._cursor.left === null) {
+            var save;
+            do {
+                save = this._cursor;
+                if(this._ancestors.length) {
+                    this._cursor = this._ancestors.pop();
+                }
+                else {
+                    this._cursor = null;
+                    break;
+                }
+            } while(this._cursor.left === save);
+        }
+        else {
+            this._ancestors.push(this._cursor);
+            this._maxNode(this._cursor.left);
+        }
+    }
+    return this._cursor !== null ? this._cursor.data : null;
+};
+
+Iterator.prototype._minNode = function(start) {
+    while(start.left !== null) {
+        this._ancestors.push(start);
+        start = start.left;
+    }
+    this._cursor = start;
+};
+
+Iterator.prototype._maxNode = function(start) {
+    while(start.right !== null) {
+        this._ancestors.push(start);
+        start = start.right;
+    }
+    this._cursor = start;
+};
+
+module.exports = TreeBase;
+
+};
+require.m['__main__'] = function(module, exports) {
+
+var TreeBase = require('./treebase');
+
+function Node(data) {
+    this.data = data;
+    this.left = null;
+    this.right = null;
+    this.red = true;
+}
+
+Node.prototype.get_child = function(dir) {
+    return dir ? this.right : this.left;
+};
+
+Node.prototype.set_child = function(dir, val) {
+    if(dir) {
+        this.right = val;
+    }
+    else {
+        this.left = val;
+    }
+};
+
+function RBTree(comparator) {
+    this._root = null;
+    this._comparator = comparator;
+    this.size = 0;
+}
+
+RBTree.prototype = new TreeBase();
+
+// returns true if inserted, false if duplicate
+RBTree.prototype.insert = function(data) {
+    var ret = false;
+
+    if(this._root === null) {
+        // empty tree
+        this._root = new Node(data);
+        ret = true;
+        this.size++;
+    }
+    else {
+        var head = new Node(undefined); // fake tree root
+
+        var dir = 0;
+        var last = 0;
+
+        // setup
+        var gp = null; // grandparent
+        var ggp = head; // grand-grand-parent
+        var p = null; // parent
+        var node = this._root;
+        ggp.right = this._root;
+
+        // search down
+        while(true) {
+            if(node === null) {
+                // insert new node at the bottom
+                node = new Node(data);
+                p.set_child(dir, node);
+                ret = true;
+                this.size++;
+            }
+            else if(is_red(node.left) && is_red(node.right)) {
+                // color flip
+                node.red = true;
+                node.left.red = false;
+                node.right.red = false;
+            }
+
+            // fix red violation
+            if(is_red(node) && is_red(p)) {
+                var dir2 = ggp.right === gp;
+
+                if(node === p.get_child(last)) {
+                    ggp.set_child(dir2, single_rotate(gp, !last));
+                }
+                else {
+                    ggp.set_child(dir2, double_rotate(gp, !last));
+                }
+            }
+
+            var cmp = this._comparator(node.data, data);
+
+            // stop if found
+            if(cmp === 0) {
+                break;
+            }
+
+            last = dir;
+            dir = cmp < 0;
+
+            // update helpers
+            if(gp !== null) {
+                ggp = gp;
+            }
+            gp = p;
+            p = node;
+            node = node.get_child(dir);
+        }
+
+        // update root
+        this._root = head.right;
+    }
+
+    // make root black
+    this._root.red = false;
+
+    return ret;
+};
+
+// returns true if removed, false if not found
+RBTree.prototype.remove = function(data) {
+    if(this._root === null) {
+        return false;
+    }
+
+    var head = new Node(undefined); // fake tree root
+    var node = head;
+    node.right = this._root;
+    var p = null; // parent
+    var gp = null; // grand parent
+    var found = null; // found item
+    var dir = 1;
+
+    while(node.get_child(dir) !== null) {
+        var last = dir;
+
+        // update helpers
+        gp = p;
+        p = node;
+        node = node.get_child(dir);
+
+        var cmp = this._comparator(data, node.data);
+
+        dir = cmp > 0;
+
+        // save found node
+        if(cmp === 0) {
+            found = node;
+        }
+
+        // push the red node down
+        if(!is_red(node) && !is_red(node.get_child(dir))) {
+            if(is_red(node.get_child(!dir))) {
+                var sr = single_rotate(node, dir);
+                p.set_child(last, sr);
+                p = sr;
+            }
+            else if(!is_red(node.get_child(!dir))) {
+                var sibling = p.get_child(!last);
+                if(sibling !== null) {
+                    if(!is_red(sibling.get_child(!last)) && !is_red(sibling.get_child(last))) {
+                        // color flip
+                        p.red = false;
+                        sibling.red = true;
+                        node.red = true;
+                    }
+                    else {
+                        var dir2 = gp.right === p;
+
+                        if(is_red(sibling.get_child(last))) {
+                            gp.set_child(dir2, double_rotate(p, last));
+                        }
+                        else if(is_red(sibling.get_child(!last))) {
+                            gp.set_child(dir2, single_rotate(p, last));
+                        }
+
+                        // ensure correct coloring
+                        var gpc = gp.get_child(dir2);
+                        gpc.red = true;
+                        node.red = true;
+                        gpc.left.red = false;
+                        gpc.right.red = false;
+                    }
+                }
+            }
+        }
+    }
+
+    // replace and remove if found
+    if(found !== null) {
+        found.data = node.data;
+        p.set_child(p.right === node, node.get_child(node.left === null));
+        this.size--;
+    }
+
+    // update root and make it black
+    this._root = head.right;
+    if(this._root !== null) {
+        this._root.red = false;
+    }
+
+    return found !== null;
+};
+
+function is_red(node) {
+    return node !== null && node.red;
+}
+
+function single_rotate(root, dir) {
+    var save = root.get_child(!dir);
+
+    root.set_child(!dir, save.get_child(dir));
+    save.set_child(dir, root);
+
+    root.red = true;
+    save.red = false;
+
+    return save;
+}
+
+function double_rotate(root, dir) {
+    root.set_child(!dir, single_rotate(root.get_child(!dir), !dir));
+    return single_rotate(root, dir);
+}
+
+module.exports = RBTree;
+};
+return require('__main__');
+})(window);
diff --git a/static/js/readcache.js b/static/js/readcache.js
index 32f5f89..2677498 100644
--- a/static/js/readcache.js
+++ b/static/js/readcache.js
@@ -52,6 +52,10 @@ var readCache = new function() {
   // A map from read ID to read object.
   var readsById = {};
 
+  // An array of RBTrees, one per track, containing all the reads assigned to
+  // that track by position.
+  var yTracks = [];
+
   /*
    * Returns all the reads in the cache.
    */
@@ -67,6 +71,15 @@ var readCache = new function() {
     start = 0;
     end = 0;
     readsById = {};
+    yTracks = [];
+  };
+
+  // Remove a specific read from the cache.
+  function removeRead(read) {
+    var removed = yTracks[read.yOrder].remove(read);
+    assert(removed);
+    assert(readsById[read.id] === read);
+    delete readsById[read.id];
   };
 
   /*
@@ -74,6 +87,32 @@ var readCache = new function() {
    * necessary.
    */
   this.setRange = function(newStart, newEnd, newBases) {
+    // Find all reads outside the new range and remove them.
+    // Ideally our RBTree implementation would just support
+    // efficiently removing ranges of nodes, or removing given an iterator.
+    for (var y = 0; y < yTracks.length; y++) {
+      var tree = yTracks[y];
+
+      // Remove nodes from the front as long as they're outside our range.
+      for(var read = tree.min(); read && read.end < newStart; read = tree.min()) {
+        assert(read.yOrder == y);
+        removeRead(read);
+      }
+
+      // Remove nodes from the end as long as they're outside our range.
+      for(var read = tree.max(); read && read.position >= newEnd; read = tree.max()) {
+        assert(read.yOrder == y);
+        removeRead(read);
+      }
+    }
+
+    // Discard stored bases if we don't want them anymore.
+    if (wantBases && !newBases) {
+      $.each(readsById, function(id, read) {
+        read.readPieces = [];
+      });
+    }
+
     // Discard any cached reads that are now entirely outside our desired window.
     $.each(readsById, function(id, read) {
       if (!overlaps(read.position, read.end, newStart, newEnd)) {
@@ -100,6 +139,16 @@ var readCache = new function() {
     return false;
   };
 
+  // Comparer function for the RBTrees
+  function readOverlapComparer(r1, r2) {
+    if (overlaps(r1.position, r1.end, r2.position, r2.end)) {
+      return 0;  // reads that overlap are considered "equal"
+    }
+    // Since we know that no two reads in the tree overlap, we can sort
+    // just by the start position.
+    return r1.position - r2.position;
+  }
+
   /*
    * Adds the supplied read to the cache if it's still relevant, assigning a
    * free yOrder property to thr read.
@@ -121,16 +170,27 @@ var readCache = new function() {
     var existingRead = readsById[read.id];
     if (existingRead) {
       read.yOrder = existingRead.yOrder;
+      var removed = yTracks[read.yOrder].remove(existingRead);
+      assert(removed);
+      var inserted = yTracks[read.yOrder].insert(read);
+      assert(inserted);
     } else {
       // Find the lowest available track for this read.
-      // TODO: Use a more efficient algorithm.
-      var trackUnavailable = [];
-      $.each(readsById, function(id, read2) {
-        if (overlaps(read.position, read.end, read2.position, read2.end)) {
-          trackUnavailable[read2.yOrder] = true;
+      assert(!('yOrder' in read));
+      for(read.yOrder = 0;; read.yOrder++) {
+        if (read.yOrder < yTracks.length) {
+          if (yTracks[read.yOrder].insert(read)) {
+            // Successfully inserted into this track.
+            break;
+          }
+        } else {
+          // Need a new track.
+          var newTree = new RBTree(readOverlapComparer);
+          yTracks[read.yOrder] = newTree;
+          var inserted = newTree.insert(read);
+          assert(inserted);
+          break;
         }
-      });
-      for(read.yOrder = 0; trackUnavailable[read.yOrder]; read.yOrder++) {
       }
     }
     readsById[read.id] = read;

From ba3fbf7d3d6118dd4905ac6f5b5118383d2438a1 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 13:46:41 -0400
Subject: [PATCH 36/42] Eliminate one extra updateDisplay call

updateDisplay is still too expensive a lot of the time.
---
 static/js/readgraph.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index bda3219..4b1b621 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -100,7 +100,7 @@ var readgraph = new function() {
       var sequenceEnd = parseInt(x.domain()[1]);
       ensureReadsCached(sequenceStart, sequenceEnd, scaleLevel > 5);
     }
-  }
+  };
 
   var moveToSequencePosition = function(position) {
     position = Math.max(0, position);
@@ -930,8 +930,6 @@ var readgraph = new function() {
     }
     if (variantParams) {
       callXhr('/api/variants', variantParams, setVariants);
-    } else {
-      setVariants([]);
     }
   };
 

From 77903c8510d34983c35196490eb9e64d764de5d1 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Thu, 10 Jul 2014 16:23:29 -0400
Subject: [PATCH 37/42] Revert accidental checkin changing appengine project
 name

---
 app.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.yaml b/app.yaml
index 70ab3ef..b1ca1c4 100644
--- a/app.yaml
+++ b/app.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-application: gabrowse-staging
+application: gabrowse
 version: 1
 runtime: python27
 api_version: 1

From a7a8aa5378c7b3e82d2f993e23ecd4fc3502367c Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Fri, 11 Jul 2014 14:44:19 -0400
Subject: [PATCH 38/42] Fix bug with bases sliding around in base view

I forgot to apply a key function to match letters to readPieces properly,
this made for some pretty funky behavior when panning around in base view.
---
 static/js/readgraph.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 4b1b621..cb5d361 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -508,10 +508,16 @@ var readgraph = new function() {
       readLetters.remove();
 
     } else if (baseView) {
-      readLetters = readLetters.data(function(read, i) {
+      var filterLetters = function(read) {
         return read.readPieces.filter(function(letter) {
           return letter.rx >= sequenceStart && letter.rx < sequenceEnd;
         });
+      };
+      readLetters = readLetters.data(filterLetters, function(letter, i) {
+        // Although the docs don't say so explicitly, it appears that the
+        // key only needs to be unique within the group (read) as opposed to
+        // across all groups.
+        return letter.rx;
       });
 
       readLetters.enter().append('text')

From 2afa222ca13e2ce8fa87a99912ddaff813f63081 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Fri, 11 Jul 2014 15:31:21 -0400
Subject: [PATCH 39/42] Use id field now that Google backends supply one.

---
 static/js/readgraph.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index cb5d361..f6dfd8f 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -752,8 +752,7 @@ var readgraph = new function() {
     var newReadIds = {};
 
     $.each(reads, function(readi, read) {
-      // TODO: Nobody is handing back actually unique ids right now
-      read.id = (read.id || read.name) + read.position + read.cigar;
+      read.id = read.id || (read.name + read.position + read.cigar);
 
       if (newReadIds[read.id]) {
         showError('There is more than one read with the ID ' + read.id +
@@ -883,7 +882,9 @@ var readgraph = new function() {
 
     if (type == READSET_TYPE) {
       var baseFields = opt_bases ? ',originalBases,baseQuality' : '';
-      queryParams.readFields = 'name,position,matePosition,mappingQuality,cigar' + baseFields;
+      // TODO: Google Read ID is rather long (increases transfer volume by ~50%
+      // in read view).  Should we synthesize our own instead?
+      queryParams.readFields = 'id,name,position,matePosition,mappingQuality,cigar' + baseFields;
     }
 
     return queryParams;

From 567fd5f23f3920e8ec99326e94bec311fc3da148 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 21 Jul 2014 09:47:11 -0400
Subject: [PATCH 40/42] Fix API errors when panning off the start/end of a
 sequence

---
 static/js/readgraph.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index f6dfd8f..4867149 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -104,7 +104,7 @@ var readgraph = new function() {
 
   var moveToSequencePosition = function(position) {
     position = Math.max(0, position);
-    position = Math.min(currentSequence['length'], position);
+    position = Math.min(currentSequence.length, position);
 
     var newX = x(position);
     newX = zoom.translate()[0] - newX + width / 2;
@@ -909,7 +909,9 @@ var readgraph = new function() {
     }
 
     var desiredStart = start - windowSize * MAX_CACHE_FACTOR;
+    desiredStart = Math.max(desiredStart, 1);
     var desiredEnd = end + windowSize * MAX_CACHE_FACTOR;
+    desiredEnd = Math.min(currentSequence.length, desiredEnd);
 
     if (overlaps(desiredStart, desiredEnd, readCache.start, readCache.end)
         && !addingBases) {

From 40af811ae1fa0d52cdb4500d2f4952261d2ba6f7 Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 21 Jul 2014 12:32:12 -0400
Subject: [PATCH 41/42] Limit panning to valid sequence positions

Also add 'clamp' helper function to simplify a few spots.
---
 static/js/readgraph.js | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 4867149..64fd875 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -70,6 +70,10 @@ var readgraph = new function() {
     return totalTracks;
   };
 
+  var clamp = function(x, min, max) {
+    return Math.min(Math.max(x, min), max);
+  };
+
   var getScaleLevel = function() {
     return Math.floor(Math.log(zoom.scale()) / Math.log(zoomLevelChange) + .1);
   };
@@ -77,11 +81,15 @@ var readgraph = new function() {
   // Called at high frequency for any navigation of the UI (not just zooming).
   // Should only do low-latency operations.
   var handleZoom = function() {
-    var tx = zoom.translate()[0];
-    // TODO: This isn't strict enough
-    tx = Math.max(tx, (1 - zoom.scale()) * width);
-    tx = Math.min(tx, 0);
-    zoom.translate([tx, 0]);
+    // Limit the domain (without changing the scale).  D3 should really
+    // handle this for us, see https://github.com/mbostock/d3/issues/1084.
+    // Note that we don't want to reset the domain itself because that
+    // also resets the scale and translation.
+    if (x.domain()[0] < 1) {
+      zoom.translate([tx - x(1) + x.range()[0], 0]);
+    } else if (x.domain()[1] > currentSequence.length) {
+      zoom.translate([tx - x(currentSequence.length) + x.range()[1], 0]);
+    }
     svg.select(".axis").call(xAxis);
 
     // Update scale bar
@@ -103,8 +111,7 @@ var readgraph = new function() {
   };
 
   var moveToSequencePosition = function(position) {
-    position = Math.max(0, position);
-    position = Math.min(currentSequence.length, position);
+    position = clamp(position, 1, currentSequence.length);
 
     var newX = x(position);
     newX = zoom.translate()[0] - newX + width / 2;
@@ -153,8 +160,7 @@ var readgraph = new function() {
 
     svg.on("mousemove", function() {
       var mouseX = d3.mouse(this)[0];
-      mouseX = Math.max(margin, mouseX);
-      mouseX = Math.min(width - margin, mouseX);
+      mouseX = clamp(mouseX, margin, width - margin);
 
       if (mouseX > width * 2/3) {
         hovertext.attr('x', mouseX - 3).style('text-anchor', 'end');
@@ -207,8 +213,7 @@ var readgraph = new function() {
       } else {
         newZoom = zoom.scale() / zoomLevelChange;
       }
-      newZoom = Math.max(1, newZoom);
-      newZoom = Math.min(maxZoom, newZoom);
+      newZoom = clamp(newZoom, 1, maxZoom);
       zoom.scale(newZoom);
       moveToSequencePosition(middleX);
     };
@@ -321,7 +326,7 @@ var readgraph = new function() {
       selectSequence(sequence);
     }
 
-    var currentLength = currentSequence['length'];
+    var currentLength = currentSequence.length;
     if (position > currentLength) {
       showError('This sequence only has ' + xFormat(currentLength) +
           ' bases. Please try a smaller position.');
@@ -378,7 +383,7 @@ var readgraph = new function() {
     }
 
     // Axis and zoom
-    x.domain([0, sequence['length']]);
+    x.domain([1, sequence['length']]);
     maxZoom = Math.ceil(Math.max(1, sequence['length'] / minRange));
     zoomLevelChange = Math.pow(maxZoom, 1/6);
     zoom.x(x).scaleExtent([1, maxZoom]).size([width, height]);
@@ -909,9 +914,9 @@ var readgraph = new function() {
     }
 
     var desiredStart = start - windowSize * MAX_CACHE_FACTOR;
-    desiredStart = Math.max(desiredStart, 1);
+    desiredStart = clamp(desiredStart, 1, currentSequence.length);
     var desiredEnd = end + windowSize * MAX_CACHE_FACTOR;
-    desiredEnd = Math.min(currentSequence.length, desiredEnd);
+    desiredEnd = clamp(desiredEnd, 1, currentSequence.length);
 
     if (overlaps(desiredStart, desiredEnd, readCache.start, readCache.end)
         && !addingBases) {

From c5bd79c8db0674876835a55eb8457b66fa360ecb Mon Sep 17 00:00:00 2001
From: Rick Byers <rbyers@chromium.org>
Date: Mon, 21 Jul 2014 12:39:57 -0400
Subject: [PATCH 42/42] Fix pan limiting typo

---
 static/js/readgraph.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/readgraph.js b/static/js/readgraph.js
index 64fd875..6672796 100644
--- a/static/js/readgraph.js
+++ b/static/js/readgraph.js
@@ -86,9 +86,9 @@ var readgraph = new function() {
     // Note that we don't want to reset the domain itself because that
     // also resets the scale and translation.
     if (x.domain()[0] < 1) {
-      zoom.translate([tx - x(1) + x.range()[0], 0]);
+      zoom.translate([zoom.translate()[0] - x(1) + x.range()[0], 0]);
     } else if (x.domain()[1] > currentSequence.length) {
-      zoom.translate([tx - x(currentSequence.length) + x.range()[1], 0]);
+      zoom.translate([zoom.translate()[0] - x(currentSequence.length) + x.range()[1], 0]);
     }
     svg.select(".axis").call(xAxis);