Merge pull request #461 from ImagingDataCommons/user/mt/multi-part-manifest

Update download manifest dialog to support fetching very large manifest (multi-chunk download, BQ, etc.) #432

Author: s-paquette

static/js/cohorts/cohort-details.js

@@ -79,6 +79,18 @@ require([
         download_manifest("json", $(this), e)
     });
 
+    $('.export-option input[type="radio"]').click(function(){
+        update_export_option($(this).attr("value"));
+    });
+
+    var update_export_option = function(export_option) {
+        $('#bq-manifest').hide();
+        $('#file-manifest').hide();
+        $('#' + export_option).show();
+    };
+
+    update_export_option("file-manifest");
+
     var download_manifest = function(file_type, clicked_button, e) {
         $('#unallowed-chars-alert').hide();
         $('#name-too-long-alert-modal').hide();
@@ -146,13 +158,24 @@ require([
            }
         });
 
+        var include_header = $('#include-header-checkbox')[0].checked;
+
         var url = BASE_URL + '/cohorts/download_manifest/' + cohort_id + '/';
         url += ("?file_type=" + file_type);
+        url += ("&include_header=" + include_header);
         url += ("&file_name=" + name);
         url += ("&header_fields=" + JSON.stringify(checked_fields));
         url += ("&columns=" + JSON.stringify(checked_columns));
         url += ("&downloadToken=" + downloadToken);
 
+        var select_box_div = $('#file-part-select-box');
+        var select_box = select_box_div.find('select');
+        if (select_box_div.is(":visible"))
+        {
+            var selected_file_part = select_box.children("option:selected").val();
+            url += ("&file_part=" + selected_file_part);
+        }
+
         location.href = url;
     };
 

static/js/image_search.js

@@ -1723,12 +1723,30 @@ require([
                 contentType: 'application/x-www-form-urlencoded',
                 success: function (data) {
                     var isFiltered = Boolean($('#search_def p').length>0);
-                    if(is_cohort) {
+                    if (is_cohort) {
                         if(data.total > 65000) {
                             $('.manifest-size-warning').show();
                         } else {
                             $('.manifest-size-warning').hide();
                         }
+
+                        var select_box_div = $('#file-part-select-box');
+                        var select_box = select_box_div.find('select');
+                        if (data.file_parts_count > 1)
+                        {
+                            select_box_div.show();
+                            for (let i = 0; i < data.file_parts_count; ++i)
+                            {
+                                select_box.append($('<option/>', {
+                                    value: i,
+                                    text : "File Part " + (i + 1)
+                                }));
+                            }
+                        }
+                        else
+                        {
+                            select_box_div.hide();
+                        }
                     } else {
                         if (isFiltered && data.total > 0){
                             $('#save-cohort-btn').prop('disabled','');
@@ -2945,6 +2963,7 @@ require([
                  // Do not disable checkboxes for export manifest dialog
                  $('.field-checkbox').removeAttr('disabled');
                  $('.column-checkbox').removeAttr('disabled');
+                 $('#include-header-checkbox').removeAttr('disabled');
 
                  $('div.ui-slider').siblings('button').prop('disabled', 'disabled');
                  $('input#hide-zeros').prop("disabled", "");
@@ -2977,7 +2996,6 @@ require([
      }
 
      $(document).ready(function () {
-
            // $('#proj_table').DataTable();
            // window.filterObj.collection_id = window.tcgaColls;
             window.selItems = new Object();

templates/cohorts/cohort_details.html

@@ -46,11 +46,6 @@ <h3 class="pull-left" role="heading" aria-level="1">Cohort Name: {{ cohort.name
                  <button id="export-manifest" class="btn btn-special pull-right" data-toggle="modal"
                         data-target="#export-manifest-modal"> Export Cohort Manifest
                  </button>
-                    <!--
-                    <a id="download-manifest" class="btn btn-default pull-right" title="Download this cohort's manifest." href="{% url 'cohort_manifest' cohort.id %}">
-                        Download Cohort Manifest
-                    </a>
-                    -->
                 </div>
             </div>
         </div>
@@ -68,8 +63,8 @@ <h3 class="pull-left" role="heading" aria-level="1">Cohort Name: {{ cohort.name
 </script>
 
 {% with is_cohort=True %}
-    {% include "idc/explore_data_core.html" %}
     {% include "cohorts/export-manifest-modal.html" %}
+    {% include "idc/explore_data_core.html" %}
 {% endwith %}
 
     <!-- Download In Progress Modal -->

templates/cohorts/export-manifest-modal.html

@@ -112,20 +112,43 @@ <h4 class="modal-title" id="exportManifestModal">Export Cohort Manifest</h4>
                                 <span class="value">GCS URL </span>
                             </label></li>
                         </div>
-                    </div>
-                </div>
+                        <br>
+
+                        <!-- Export Option -->
+                        <div class="export-option">
+                            <span class="value"><input type="radio" name="colorRadio" value="file-manifest" checked> Files</span>
+                            <span style="margin-left:10px" class="value"><input type="radio" name="colorRadio" value="bq-manifest"> BigQuery</span>
+                        </div>
+                        <br>
 
-                <div class="modal-footer">
-                    {% csrf_token %}
-                    <a id="download-csv" class="btn btn-primary" title="Download this cohort's manifest in CSV format.">
-                        Download CSV
-                    </a>
-                    <a id="download-tsv" class="btn btn-primary " title="Download this cohort's manifest in TSV format.">
-                        Download TSV
-                    </a>
-                    <a id="download-json" class="btn btn-primary " title="Download this cohort's manifest in JSON format.">
-                        Download JSON
-                    </a>
+                        <div id="bq-manifest">
+                            <a id="get-bq-table" class="btn btn-primary" title="Get BQ Table of this manifest.">
+                               Get BQ Table
+                            </a>
+                        </div>
+                        <div id="file-manifest">
+                            <div id="file-part-select-box">
+                                <select class="form-control">
+                                </select>
+                                <br>
+                            </div>
+
+                            <span><input id="include-header-checkbox" type="checkbox" checked> Include header fields (CSV and TSV only)</span>
+                            <br>
+                            <br>
+
+                            {% csrf_token %}
+                            <a id="download-csv" class="btn btn-primary" title="Download this cohort's manifest in CSV format.">
+                                Download CSV
+                            </a>
+                            <a id="download-tsv" class="btn btn-primary " title="Download this cohort's manifest in TSV format.">
+                                Download TSV
+                            </a>
+                            <a id="download-json" class="btn btn-primary " title="Download this cohort's manifest in JSON format.">
+                                Download JSON
+                            </a>
+                        </div>
+                    </div>
                 </div>
             </form>
         </div>

templates/idc/landing.html

@@ -122,6 +122,7 @@
             </div>
         </div>
         {% include 'share/site_footer.html'  %}
+
     </div>
 {% endblock %}