Merge pull request #139 from biothings/validation

Validation Feature

Author: DylanWelzel

webapp/src/BaseDataSource.vue

@@ -5,21 +5,22 @@
 import axios from 'axios'
 import bus from './bus.js'
 
-export default {
+export default {
   name: 'base-data-source',
   // Note: we don't declare "source", it must be defined in subclass/mixed
   // (sometimes it's a prop, sometimes it's a data field
-  created () {
+  created() {
     bus.$on('change_source', this.onSourceChanged)
   },
-  beforeDestroy () {
+  beforeDestroy() {
     bus.$off('change_source', this.onSourceChanged)
     $('.ui.basic.unregister.modal').remove()
   },
-  data () {
+  data() {
     return {
       limit_error: null,
-      sample_error: null
+      sample_error: null,
+      validations: {}
     }
   },
   computed: {
@@ -29,6 +30,9 @@ export default {
     upload_status: function () {
       return this.getStatus('upload')
     },
+    validate_status: function () {
+      return this.getStatus('validate')
+    },
     download_status: function () {
       if (this.source.download && this.source.download.status) { return this.source.download.status } else { return 'unknown status' }
     },
@@ -52,6 +56,9 @@ export default {
     upload_error: function () {
       return this.getError('upload')
     },
+    validate_error: function () {
+      return this.getError('validate')
+    },
     download_error: function () {
       if (this.source.download && this.source.download.error) { return this.source.download.error }
     },
@@ -74,7 +81,7 @@ export default {
       var status = 'unknown'
       if (this.source.hasOwnProperty(subkey)) {
         for (var subsrc in this.source[subkey].sources) {
-          if (['failed', 'inspecting', 'uploading'].indexOf(this.source[subkey].sources[subsrc].status) != -1) {
+          if (['failed', 'inspecting', 'uploading', 'validating'].indexOf(this.source[subkey].sources[subsrc].status) != -1) {
             status = this.source[subkey].sources[subsrc].status
             // precedence to these statuses
             break
@@ -97,6 +104,7 @@ export default {
       if (this.download_error) { errs.push('Dump') }
       if (this.upload_error.length) { errs.push('Upload') }
       if (this.inspect_error.length) { errs.push('Inspect') }
+      if (this.validate_error.length) { errs.push('Validate') }
       return errs.join(' - ') + ' error'
     },
     dump: function (release = null, force = null) {
@@ -111,13 +119,16 @@ export default {
           console.log('Error getting job manager information: ' + err)
         })
     },
-    upload: function (subsrc = null, release = null){
+    upload: function (subsrc = null, release = null, validate = false) {
       var srcname = this.source.name
       if (subsrc != null) { srcname += '.' + subsrc } // upload a sub-source only
       let payload = {};
       if (release && release.trim() !== '') {
         payload.release = release;
       }
+      if (validate) {
+        payload.validate = true;
+      }
       axios.put(axios.defaults.baseURL + `/source/${srcname}/upload`, payload)
         .then(response => {
           console.log(response.data.result)
@@ -126,6 +137,51 @@ export default {
           console.log('Error getting job manager information: ' + err)
         })
     },
+    createValidation(subsrc = null) {
+      var srcname = this.source.name
+      if (subsrc != null) srcname += '.' + subsrc
+      return axios.put(axios.defaults.baseURL + `/source/${srcname}/create_validation`)
+        .then(response => {
+          // Step 1: The create request finished
+          // Step 2: Now fetch the updated validations
+          return this.getValidations(subsrc)
+        })
+        .catch(err => {
+          console.error('Error creating validation:', err)
+          throw err
+        })
+    },
+    validate: function (subsrc = null, model_file = null) {
+      var srcname = this.source.name
+      if (subsrc != null) { srcname += '.' + subsrc } // validate a sub-source only
+      let payload = {};
+      if (model_file && model_file.trim() !== '') {
+        payload.model_file = model_file;
+      }
+      axios.post(axios.defaults.baseURL + `/source/${srcname}/validate`, payload)
+        .then(response => {
+          // console.log(response.data.result)
+        })
+        .catch(err => {
+          console.log('Error getting job manager information: ' + err)
+        })
+    },
+    getValidations: function (subsrc = null) {
+      var srcname = this.source.name
+      if (subsrc != null) {
+        srcname += '.' + subsrc
+      }
+      return axios.get(axios.defaults.baseURL + `/source/${srcname}/validations`)
+        .then(response => {
+          this.$set(this.validations, subsrc, response.data.result)
+          // console.log(response.data.result)
+          return response.data.result
+        })
+        .catch(err => {
+          console.log('Error getting validations information:', err)
+          throw err
+        })
+    },
     unregister: function () {
       $(`.${this.source.name}.ui.basic.unregister.modal`)
         .modal('setting', {
@@ -147,8 +203,8 @@ export default {
     inspect: function () {
       bus.$emit('do_inspect', ['src', this.source._id])
     },
-    mark_dump_success (dry_run=false, dry_run_callback) {
-      axios.put(axios.defaults.baseURL + `/source/${this.source.name}/mark_dump_success`, {dry_run: dry_run})
+    mark_dump_success(dry_run = false, dry_run_callback) {
+      axios.put(axios.defaults.baseURL + `/source/${this.source.name}/mark_dump_success`, { dry_run: dry_run })
         .then(response => {
           if (dry_run && dry_run_callback) {
             dry_run_callback(response.data.result)
@@ -159,7 +215,7 @@ export default {
           console.log('Error getting job manager information: ' + err)
         })
     },
-    onSourceChanged (_id = null, op = null) {
+    onSourceChanged(_id = null, op = null) {
       // this method acts as a dispatcher, reacting to change_source events, filtering
       // them for the proper source
       // _id null: event containing change about a source but we don't know which one

webapp/src/DataInspection.vue

@@ -2,7 +2,7 @@
 
   <div class="ui tab" :data-tab="source_name + '-type-stats'">
     <p>
-      This is the field type and stats for <b>{{page_type}}</b>.
+      This is the field type and stats for <b>{{ page_type }}</b>.
     </p>
     <p>
       It provides a summary of the data structure,
@@ -34,12 +34,8 @@
         <tbody>
           <tr v-for="row in inspection_data">
             <td>
-              <div v-if="hasInspectionFieldValidationWarnings(row)"
-                class="tooltip"
-                data-position="top left"
-                data-variation="very wide"
-                :data-html="formatInspectionValidationTooltipMessage(row.warnings)"
-              >
+              <div v-if="hasInspectionFieldValidationWarnings(row)" class="tooltip" data-position="top left"
+                data-variation="very wide" :data-html="formatInspectionValidationTooltipMessage(row.warnings)">
                 <span class="ui text warning">{{ row.field_name }} <i class="exclamation circle icon"></i></span>
               </div>
               <div v-else>{{ row.field_name }}</div>
@@ -76,10 +72,10 @@ export default {
     'source_name',
     'source_data',
   ],
-  mounted () {
+  mounted() {
     this.fetch_flatten_inspection_data()
   },
-  data () {
+  data() {
     return {
       inspection_data: [],
     }
@@ -103,29 +99,29 @@ export default {
       }
 
       axios.put(axios.defaults.baseURL + '/flatten_inspection_data', data)
-      .then(response => {
-        console.log(response.data.result)
-        if (response.data.result?.[self.source_name]) {
-          const inspection_data_by_modes = response.data.result[self.source_name]
-          self.inspection_data = inspection_data_by_modes.type || []
-          if (Object.keys(inspection_data_by_modes.stats).length > 1) {
-            self.inspection_data = inspection_data_by_modes.stats
+        .then(response => {
+          // console.log(response.data.result)
+          if (response.data.result?.[self.source_name]) {
+            const inspection_data_by_modes = response.data.result[self.source_name]
+            self.inspection_data = inspection_data_by_modes.type || []
+            if (Object.keys(inspection_data_by_modes.stats).length > 1) {
+              self.inspection_data = inspection_data_by_modes.stats
+            }
           }
-        }
-        else {
-          self.inspection_data = []
-        }
-        setTimeout(() => {$('.tooltip').popup()}, 0)
-      })
-      .catch(err => {
-        console.log('Error fetching flatten inspection data: ' + err)
-      })
+          else {
+            self.inspection_data = []
+          }
+          setTimeout(() => { $('.tooltip').popup() }, 0)
+        })
+        .catch(err => {
+          console.log('Error fetching flatten inspection data: ' + err)
+        })
     },
     hasInspectionFieldValidationWarnings: function (field_inspection) {
       return field_inspection.warnings.length > 0
     },
     hasInspectionValidationWarnings: function (inspection_data) {
-      for (const field_inspection of inspection_data) {        
+      for (const field_inspection of inspection_data) {
         if (this.hasInspectionFieldValidationWarnings(field_inspection)) {
           return true
         }
@@ -151,7 +147,7 @@ export default {
 .sortable .row {
   padding-top: 0.2rem;
   padding-bottom: 0.2rem;
-  border: 1px solid rgba(34,36,38,.1);
+  border: 1px solid rgba(34, 36, 38, .1);
 }
 
 .sortable .column {