Skip to content

Commit ef27b62

Browse files
higorspintohigorspinto
committed
Merge branch 'download-option' into 'master'
[Data Explorer non capped] Explicitly display file download format See merge request datopian/clients/nhs-dataexplorer!23
2 parents 3c8511b + 0129df6 commit ef27b62

File tree

3 files changed

+91
-39
lines changed

3 files changed

+91
-39
lines changed

‎ckanext/dataexplorer/plugin.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
natural_number_validator = p.toolkit.get_validator('natural_number_validator')
1313
Invalid = p.toolkit.Invalid
1414

15+
def get_datastore_search_rows_max():
16+
return config.get('ckan.datastore_search_rows_max', 0)
17+
1518

1619
def get_mapview_config():
1720
'''
@@ -97,7 +100,9 @@ def view_template(self, context, data_dict):
97100

98101
def get_helpers(self):
99102
return {
100-
'get_map_config': get_mapview_config
103+
'get_map_config': get_mapview_config,
104+
'get_datastore_search_rows_max': get_datastore_search_rows_max,
105+
101106
}
102107

103108

‎ckanext/dataexplorer/public/vendor/recline/recline.js‎

Lines changed: 84 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4762,8 +4762,7 @@ this.recline.View = this.recline.View || {};
47624762
<label>' +
47634763
ckan.i18n._("Format") +
47644764
'</label> \
4765-
<select class="select-format form-control"> \
4766-
<option value="csv">CSV</option> \
4765+
<select id="download-format" class="select-format form-control"> \
47674766
</select> \
47684767
<input class="form-control extract-data-input" type="hidden" id="extract_data" name="extract_data" value=""> \
47694768
</fieldset> \
@@ -4781,38 +4780,54 @@ this.recline.View = this.recline.View || {};
47814780
"click .extract-button": "onExtract",
47824781
},
47834782
initialize: function () {
4783+
var self = this;
4784+
const DATASTORE_SEARCH_ROWS_MAX = 32000; //TODO: Get param from environment/backend
4785+
// console.log(self.options);
47844786
_.bindAll(this, "render");
47854787
this.render();
4788+
//Timeout of 2 seconds ensures that the form has been rendered before the select element is accessed
4789+
setTimeout(() => {
4790+
document.getElementById("download-format").innerHTML =
4791+
DATASTORE_SEARCH_ROWS_MAX <= self.model.recordCount
4792+
? '<option value="compressed-csv">Compressed CSV</option>'
4793+
: '<option value="csv">CSV</option><option value="compressed-csv">Compressed CSV</option><option value="json">JSON</option>';
4794+
}, 2000);
4795+
4796+
//Pre-Load helper libraries JSZip for zipping files
4797+
$.getScript(
4798+
"https://cdnjs.cloudflare.com/ajax/libs/jszip/3.5.0/jszip.min.js"
4799+
);
4800+
4801+
$.ajaxSetup({
4802+
cache: true,
4803+
});
47864804
},
47874805
render: function () {
47884806
var self = this;
4789-
47904807
var out = Mustache.render(this.template, {});
47914808
this.$el.html(out);
47924809
},
47934810
onExtract: function (e) {
47944811
var self = this;
47954812
e.preventDefault();
47964813

4797-
// var format = this.$el.find(".select-format").val();
4814+
var format = document.getElementById("download-format").value;
47984815
var fields = self.model.queryState.attributes.fields;
47994816
var query = CKAN._normalizeQuery(self.model.queryState.attributes);
48004817
query.ckan_resource_id = self.model.attributes.id;
48014818
query.resource_id = self.model.attributes.bq_table_name;
48024819
query.limit = this.model.recordCount;
4803-
// query.format = format;
4804-
query.format = "csv";
4805-
48064820
query.offset = 0;
48074821
query.fields = fields;
4822+
48084823
var input = this.$el
48094824
.find(".extract-data-input")
48104825
.val(JSON.stringify(query));
48114826

48124827
var sql_query = this.jsQueryToSQL(query);
4813-
this.extractFile(self, sql_query);
4828+
this.extractFile(self, sql_query, format);
48144829
},
4815-
extractFile: function (self, sql_query) {
4830+
extractFile: function (self, sql_query, format) {
48164831
var base_path = self.model.attributes.endpoint || self.options.site_url;
48174832
//console.log(base_path);
48184833
var endpoint = `${base_path}/3/action/datastore_search_sql?sql=${sql_query}`; // USE BASE_PATH IN PRODUCTION
@@ -4829,9 +4844,9 @@ this.recline.View = this.recline.View || {};
48294844
});
48304845
self.progress(true);
48314846
} else {
4832-
//small file, convert json result to CSV
4833-
this.exportCSVFile(
4847+
this.exportFile(
48344848
resource.result.result.records,
4849+
format,
48354850
self.model.attributes.title,
48364851
self
48374852
);
@@ -4847,42 +4862,75 @@ this.recline.View = this.recline.View || {};
48474862
this.showErrorModal();
48484863
});
48494864
},
4850-
exportCSVFile: function (resp_json, filename, self) {
4851-
var exported_filename = filename + ".csv";
4865+
exportFile: function (resp_json, format, filename, self) {
48524866
var src = "https://unpkg.com/[email protected]/papaparse.min.js";
48534867
$.getScript(src, function () {
48544868
try {
4855-
let csv = Papa.unparse(resp_json);
4856-
var blob = new Blob([csv], {
4857-
type: "text/csv;charset=utf-8;",
4858-
});
4859-
if (navigator.msSaveBlob) {
4860-
// IE 10+
4861-
navigator.msSaveBlob(blob, exported_filename);
4869+
let blob = null;
4870+
let exported_filename = "";
4871+
4872+
if (format === "csv") {
4873+
exported_filename = filename + ".csv";
4874+
let csv = Papa.unparse(resp_json);
4875+
blob = new Blob([csv], {
4876+
type: "text/csv;charset=utf-8;",
4877+
});
4878+
self.downloadBlob(blob, exported_filename)
4879+
self.progress(true);
4880+
4881+
} else if (format === "json") {
4882+
exported_filename = filename + ".json";
4883+
let json = JSON.stringify(resp_json);
4884+
blob = new Blob([json], {
4885+
type: "text/plain;charset=utf-8;",
4886+
});
4887+
self.downloadBlob(blob, exported_filename)
4888+
self.progress(true);
4889+
48624890
} else {
4863-
var link = document.createElement("a");
4864-
if (link.download !== undefined) {
4865-
// Browsers that support HTML5 download attribute
4866-
var url = URL.createObjectURL(blob);
4867-
link.setAttribute("href", url);
4868-
link.setAttribute("download", exported_filename);
4869-
link.style.visibility = "hidden";
4870-
document.body.appendChild(link);
4871-
link.click();
4872-
document.body.removeChild(link);
4873-
}
4891+
// zip files
4892+
exported_filename = filename + ".csv";
4893+
let csv = Papa.unparse(resp_json);
4894+
let blob_content = new Blob([csv], {
4895+
type: "text/csv;charset=utf-8;",
4896+
});
4897+
let zip = new JSZip();
4898+
zip.file(exported_filename, blob_content);
4899+
zip
4900+
.generateAsync({
4901+
type:"blob",
4902+
})
4903+
.then(function (zipped_blob) {
4904+
self.downloadBlob(zipped_blob, filename)
4905+
self.progress(true);
4906+
});
48744907
}
4875-
self.progress(true);
4908+
48764909
} catch (error) {
48774910
console.warn(error);
48784911
self.progress(true);
48794912
self.showErrorModal();
48804913
}
48814914
});
4882-
//caches Papaparse
4883-
$.ajaxSetup({
4884-
cache: true,
4885-
});
4915+
4916+
},
4917+
downloadBlob : function (blob, exported_filename){
4918+
if (navigator.msSaveBlob) {
4919+
// IE 10+
4920+
navigator.msSaveBlob(blob, exported_filename);
4921+
} else {
4922+
var link = document.createElement("a");
4923+
if (link.download !== undefined) {
4924+
// Browsers that support HTML5 download attribute
4925+
var url = URL.createObjectURL(blob);
4926+
link.setAttribute("href", url);
4927+
link.setAttribute("download", exported_filename);
4928+
link.style.visibility = "hidden";
4929+
document.body.appendChild(link);
4930+
link.click();
4931+
document.body.removeChild(link);
4932+
}
4933+
}
48864934
},
48874935
showErrorModal: function () {
48884936
var modal = document.getElementsByClassName("modal")[0];
@@ -5006,13 +5054,11 @@ this.recline.View = this.recline.View || {};
50065054
return where_str;
50075055
},
50085056
get_field_type: function (value) {
5009-
50105057
if (isNaN(Number(value))) {
50115058
return "string";
50125059
} else {
50135060
return "num";
50145061
}
5015-
50165062
},
50175063
});
50185064
})(jQuery, recline.View);

‎ckanext/dataexplorer/templates/recline_view.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
data-module-resource = "{{ h.dump_json(resource_json) }}";
1010
data-module-resource-view = "{{ h.dump_json(resource_view_json) }}";
1111
data-module-map_config= "{{ h.dump_json(map_config) }}";
12+
data-module-datastore_search_rows_max= "{{ h.get_datastore_search_rows_max() }}"
1213
>
1314
<h4 class="loading-dialog">
1415
<div class="loading-spinner"></div>

0 commit comments

Comments
 (0)