Individual submission & entity geodata endpoints (#1622)

Individual submission & entity geodata endpoints

Author: brontolosone

lib/constants.js

@@ -43,6 +43,10 @@ const Constants = {
 
   // null means "none of the other statuses"
   entityConflictStates: new Set(['soft', 'hard', null]),
+
+  // Just the general format, no version validation, and it's fine if the reserved bits are set.
+  // That's fine for PostgreSQL too.
+  UUIDRegex: new RegExp(['^', '{8}-', '{4}-', '{4}-', '{4}-', '{12}$'].join('[0-9a-f]'), 'i'),
 };
 
 module.exports = { Constants };

lib/http/service.js

@@ -98,6 +98,7 @@ module.exports = (container) => {
   require('../resources/forms')(service, endpoint, anonymousEndpoint);
   require('../resources/users')(service, endpoint, anonymousEndpoint);
   require('../resources/sessions')(service, endpoint, anonymousEndpoint);
+  require('../resources/geo-extracts')(service, endpoint);
   require('../resources/submissions')(service, endpoint, anonymousEndpoint);
   require('../resources/config')(service, endpoint);
   require('../resources/projects')(service, endpoint);
@@ -112,7 +113,6 @@ module.exports = (container) => {
   require('../resources/entities')(service, endpoint);
   require('../resources/oidc')(service, endpoint, anonymousEndpoint);
   require('../resources/user-preferences')(service, endpoint);
-  require('../resources/geo-extracts')(service, endpoint);
 
   ////////////////////////////////////////////////////////////////////////////////
   // POSTRESOURCE HANDLERS

lib/model/query/geo-extracts.js

@@ -8,6 +8,7 @@
 // except according to the terms contained in the LICENSE file.
 
 const { sql } = require('slonik');
+const { arrayHasElements } = require('../../util/util');
 
 
 const stateFilterToQueryFragments = (field, states, pgType = 'text') => {
@@ -17,37 +18,36 @@ const stateFilterToQueryFragments = (field, states, pgType = 'text') => {
   const theFilters = [alsoNullFilter, alsoOtherStatesFilter].filter(el => el !== null);
   switch (theFilters.length) {
     case 1:
-      return sql`AND ${theFilters[0]}`;
+      return sql`${theFilters[0]}`;
     case 2:
-      return sql`AND ${theFilters[0]} OR ${theFilters[1]}`;
+      return sql`${theFilters[0]} OR ${theFilters[1]}`;
     default:
       return sql``;
   }
 };
 
-const getSubmissionFeatureCollectionGeoJson = (formPK, fieldPaths, submitterIds, TSTZRange, reviewStates, deleted, limit) => ({ db }) => {
-
-  const formFieldFilter = fieldPaths.length ? sql`AND ffgeo.path = ANY (${sql.array(fieldPaths, 'text')})` : sql`AND ffgeo.is_default`;
-  const doPathInProperties = fieldPaths.length ? sql`, 'properties', NULL` : sql`, 'properties', json_build_object('fieldpath', path)`;
-  const doAggregateByPath = fieldPaths.length ? sql`` : sql`, path`;
-  const submitterFilter = submitterIds.length ? sql`AND sdef."submitterId" = ANY(${sql.array(submitterIds, 'int4')})` : sql``;
-  const TSTZRangeFilter = TSTZRange ? sql`AND sdef."createdAt" <@ tstzrange(${TSTZRange[0]}, ${TSTZRange[1]}, ${TSTZRange[2]})` : sql``;
-  const reviewStateFilter = reviewStates.length ? stateFilterToQueryFragments(['sub', 'reviewState'], reviewStates) : sql``;
-  const deletedFilter = deleted ? sql`IS NOT NULL` : sql`IS NULL`;
+const getSubmissionFeatureCollectionGeoJson = (formPK, IDs, fieldPaths, submitterIds, TSTZRange, reviewStates, deleted, limit, onlyCurrent=true, assumeRootSubmissionId=true) => ({ db }) => {
+  const idFilterPredicate = sql`AND ${sql.identifier([onlyCurrent ? 'sub' : 'sdef', 'instanceId'])} = ANY(${sql.array(IDs, 'text')})`;
+  const submissionIdFilter = (onlyCurrent && arrayHasElements(IDs)) ? idFilterPredicate : sql``;
+  const submissionDefIdFilter = (!onlyCurrent && arrayHasElements(IDs)) ? idFilterPredicate : sql``;
+  const onlyCurrentFilter = onlyCurrent ? sql`AND sdef.current` : sql``;
+  const formFieldFilter = arrayHasElements(fieldPaths) ? (fieldPaths.includes('all') ? sql`` : sql`AND ffgeo.path = ANY (${sql.array(fieldPaths, 'text')})`) : sql`AND ffgeo.is_default`;
+  const doPathInProperties = arrayHasElements(fieldPaths) ? sql`, 'properties', NULL` : sql`, 'properties', json_build_object('fieldpath', path)`;
+  const doAggregateByPath = arrayHasElements(fieldPaths) ? sql`` : sql`, path`;
+  const submitterFilter = arrayHasElements(submitterIds) ? sql`AND sdef."submitterId" = ANY(${sql.array(submitterIds, 'int4')})` : sql``;
+  const TSTZRangeFilter = arrayHasElements(TSTZRange) ? sql`AND sdef."createdAt" <@ tstzrange(${TSTZRange[0]}, ${TSTZRange[1]}, ${TSTZRange[2]})` : sql``;
+  const reviewStateFilter = arrayHasElements(reviewStates) ? sql`AND ${stateFilterToQueryFragments(['sub', 'reviewState'], reviewStates)}` : sql``;
+  const deletedFilter = deleted ? sql`AND sub."deletedAt" IS NOT NULL` : sql`AND sub."deletedAt" IS NULL`;
   const queryLimit = limit ? sql`LIMIT ${limit}` : sql``;
-
+  // data coming from revisions are in some cases expected to take on the instanceId of the root of the edit-lineage. And in some other cases, not.
+  const reportedInstanceIdPicker = assumeRootSubmissionId ? sql`CASE WHEN root THEN sdef."instanceId" ELSE sub."instanceId" END as "instanceId"` : sql`sdef."instanceId"`;
   return db.oneFirst(sql`
         -- all targeted submission fields, with info on whether the geojson-value is cached
         WITH geo_needed AS (
             SELECT
                 sxg.submission_def_id IS NOT NULL as is_cached,
                 sdef.id as submission_def_id,
-                CASE
-                    WHEN root
-                        THEN sdef."instanceId"
-                    ELSE
-                        sub."instanceId"  -- revisions take on the instanceId of the root of the edit-lineage
-                END as "instanceId",
+                ${reportedInstanceIdPicker},
                 ffgeo.type,
                 ffgeo.fieldhash,
                 ffgeo.repeatgroup_cardinality,
@@ -60,15 +60,15 @@ const getSubmissionFeatureCollectionGeoJson = (formPK, fieldPaths, submitterIds,
                     fdef."formId" = ${formPK}
                     AND
                     sdef."formDefId" = fdef.id
-                    AND
-                    sdef.current
+                    ${submissionDefIdFilter}
+                    ${onlyCurrentFilter}
                     ${submitterFilter}
                     ${TSTZRangeFilter}
                 )
                 INNER JOIN submissions sub ON (
                     sdef."submissionId" = sub.id
-                    AND
-                    sub."deletedAt" ${deletedFilter}
+                    ${submissionIdFilter}
+                    ${deletedFilter}
                     ${reviewStateFilter}
                 )
                 INNER JOIN form_field_geo ffgeo ON (
@@ -183,11 +183,12 @@ const getSubmissionFeatureCollectionGeoJson = (formPK, fieldPaths, submitterIds,
 };
 
 
-const getEntityFeatureCollectionGeoJson = (datasetPK, creatorIds, TSTZRange, conflictStates, deleted, limit) => ({ db }) => {
+const getEntityFeatureCollectionGeoJson = (datasetPK, IDs, creatorIds, TSTZRange, conflictStates, deleted, limit) => ({ db }) => {
 
-  const creatorFilter = creatorIds.length ? sql`AND e."creatorId" = ANY(${sql.array(creatorIds, 'int4')})` : sql``;
-  const TSTZRangeFilter = TSTZRange ? sql`AND e."createdAt" <@ tstzrange(${TSTZRange[0]}, ${TSTZRange[1]}, ${TSTZRange[2]})` : sql``;
-  const conflictStatusFilter = conflictStates.length ? stateFilterToQueryFragments(['e', 'conflict'], conflictStates, 'conflictType') : sql``;
+  const idFilter = arrayHasElements(IDs) ? sql`AND e.uuid = ANY(${sql.array(IDs, 'text')})` : sql``; // TODO: this should be a uuid[] array once/if the entities."uuid" column is converted from varchar(255) to uuid
+  const creatorFilter = arrayHasElements(creatorIds) ? sql`AND e."creatorId" = ANY(${sql.array(creatorIds, 'int4')})` : sql``;
+  const TSTZRangeFilter = arrayHasElements(TSTZRange) ? sql`AND e."createdAt" <@ tstzrange(${TSTZRange[0]}, ${TSTZRange[1]}, ${TSTZRange[2]})` : sql``;
+  const conflictStatusFilter = arrayHasElements(conflictStates) ? sql`AND ${stateFilterToQueryFragments(['e', 'conflict'], conflictStates, 'conflictType')}` : sql``;
   const deletedFilter = deleted ? sql`IS NOT NULL` : sql`IS NULL`;
   const queryLimit = limit ? sql`LIMIT ${limit}` : sql``;
 
@@ -208,6 +209,7 @@ const getEntityFeatureCollectionGeoJson = (datasetPK, creatorIds, TSTZRange, con
                   e."datasetId" = ds.id
                   AND
                   e."deletedAt" ${deletedFilter}
+                  ${idFilter}
                   ${creatorFilter}
                   ${TSTZRangeFilter}
                   ${conflictStatusFilter}

lib/model/query/submissions.js

@@ -385,6 +385,17 @@ join submission_defs on submission_defs."instanceId"=${instanceId}
 where "formId"=${formId} and draft=${draft} and submissions."deletedAt" is null`)
   .then(map((row) => row.instanceId));
 
+const defExists = (formId, submissionInstanceId, versionInstanceId = null) => (db) => db.oneFirst(
+  sql`SELECT EXISTS(
+    SELECT 1
+    FROM submissions sub
+    INNER JOIN submission_defs def ON (
+      sub."deletedAt" IS NULL
+      AND
+      (sub."formId", sub.id, sub."instanceId", def."instanceId") = (${formId}, def."submissionId", ${submissionInstanceId}, ${(versionInstanceId || submissionInstanceId)})
+    )
+  )`
+);
 
 ////////////////////////////////////////////////////////////////////////////////
 // EXPORT
@@ -532,7 +543,7 @@ module.exports = {
   setSelectMultipleValues, getSelectMultipleValuesForExport,
   getByIdsWithDef, getSubAndDefById,
   getByIds, getAllForFormByIds, getById, countByFormId, verifyVersion,
-  getDefById, getCurrentDefByIds, getCurrentDefColByIds, getCurrentDefColsByIds, getAnyDefByFormAndInstanceId, getDefsByFormAndLogicalId, getDefBySubmissionAndInstanceId, getRootForInstanceId,
+  getDefById, getCurrentDefByIds, getCurrentDefColByIds, getCurrentDefColsByIds, getAnyDefByFormAndInstanceId, getDefsByFormAndLogicalId, getDefBySubmissionAndInstanceId, getRootForInstanceId, defExists,
   getDeleted,
   streamForExport, getForExport
 };

lib/resources/entities.js

@@ -8,11 +8,12 @@
 // except according to the terms contained in the LICENSE file.
 
 const { getOrNotFound, reject } = require('../util/promise');
-const { isTrue, success } = require('../util/http');
+const { isTrue, success, json } = require('../util/http');
 const { Entity } = require('../model/frames');
 const Problem = require('../util/problem');
 const { diffEntityData, extractBulkSource, getWithConflictDetails } = require('../data/entity');
 const { QueryOptions } = require('../util/db');
+const { Constants } = require('../constants');
 
 module.exports = (service, endpoint) => {
 
@@ -87,6 +88,29 @@ module.exports = (service, endpoint) => {
 
   }));
 
+
+  service.get('/projects/:projectId/datasets/:name/entities/:uuid/geojson', endpoint.plain(async ({ Datasets, Entities, GeoExtracts }, { params, auth }) => {
+
+    // Saves a query and gives informative feedback if we do this straight away
+    if (!Constants.UUIDRegex.test(params.uuid)) {
+      throw Problem.user.unexpectedValue({
+        field: 'URL UUID path component',
+        value: params.uuid,
+        reason: `is not interpretable as a UUID: ${params.uuid}`,
+      });
+    }
+
+    const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound);
+    await auth.canOrReject('entity.read', dataset);
+
+    // We need to get the entity itself to see if it exists (and if not, return a 404).
+    // That's because when getting the geodata, we can't distinguish between the entity not having any geodata (or having invalid geodata)
+    // and the entity itself not existing.
+    await Entities.getById(dataset.id, params.uuid).then(getOrNotFound);
+    return GeoExtracts.getEntityFeatureCollectionGeoJson(dataset.id, [params.uuid]).then(json);
+  }));
+
+
   // Create a single entity or bulk create multiple entities.
   // In either case, this appends new entities to a dataset.
   service.post('/projects/:id/datasets/:name/entities', endpoint(async ({ Datasets, Entities }, { auth, body, params, userAgent }) => {

lib/resources/geo-extracts.js

@@ -11,10 +11,32 @@ const { getOrNotFound } = require('../util/promise');
 const { Form } = require('../model/frames');
 const { Sanitize } = require('../util/param-sanitize');
 const { isTrue, json } = require('../util/http');
+const Problem = require('../util/problem');
 
 
+const getSingleGeo = async ({ Forms, Submissions, GeoExtracts }, { params, auth, query }, rootId, versionId) => {
+  const form = await Forms.getByProjectAndXmlFormId(params.projectId, params.xmlFormId, Form.WithoutDef, Form.WithoutXml)
+    .then(getOrNotFound)
+    .then((foundForm) => auth.canOrReject('submission.read', foundForm));
+  // we need to distinguish between submission-does-not-exist and submission-exists-but-has-no-(valid-)geodata, hence this probe.
+  if (!await Submissions.defExists(form.id, rootId, versionId)) throw Problem.user.notFound();
+  return GeoExtracts.getSubmissionFeatureCollectionGeoJson(form.id, [versionId || rootId], Sanitize.queryParamToArray(query.fieldpath), null, null, null, false, null, !versionId, false)
+    .then(json);
+};
+
 module.exports = (service, endpoint) => {
 
+  // Single-submission endpoints
+  service.get(`/projects/:projectId/forms/:xmlFormId/submissions/:instanceId.geojson`, endpoint.plain(async (container, context) =>
+    getSingleGeo(container, context, context.params.instanceId, null)
+  ));
+
+  service.get(`/projects/:projectId/forms/:xmlFormId/submissions/:rootId/versions/:instanceId.geojson`, endpoint.plain(async (container, context) =>
+    getSingleGeo(container, context, context.params.rootId, context.params.instanceId)
+  ));
+
+
+  // Bulk endpoints
   service.get('/projects/:projectId/forms/:xmlFormId/submissions.geojson', endpoint.plain(async ({ Forms, GeoExtracts }, { auth, query, params }) => {
 
     const form = await Forms.getByProjectAndXmlFormId(params.projectId, params.xmlFormId, Form.WithoutDef, Form.WithoutXml)
@@ -23,6 +45,7 @@ module.exports = (service, endpoint) => {
 
     return GeoExtracts.getSubmissionFeatureCollectionGeoJson(
       form.id,
+      Sanitize.queryParamToArray(query.submissionID),
       Sanitize.queryParamToArray(query.fieldpath),
       Sanitize.queryParamToIntArray(query.submitterId, 'submitterId'),
       Sanitize.getTSTZRangeFromQueryParams(query),
@@ -40,6 +63,7 @@ module.exports = (service, endpoint) => {
 
     return GeoExtracts.getEntityFeatureCollectionGeoJson(
       foundDataset.id,
+      Sanitize.queryParamToUuidArray(query.entityUUID, 'entityUUID'),
       Sanitize.queryParamToIntArray(query.creatorId, 'creatorId'),
       Sanitize.getTSTZRangeFromQueryParams(query),
       Sanitize.getEntityConflictStates(query.conflict, 'conflict'),

lib/util/param-sanitize.js

@@ -57,6 +57,22 @@ class Sanitize {
   }
 
 
+  static queryParamToUuidArray(queryParam, queryParamName) {
+    return this.queryParamToArray(queryParam).map(el => {
+      const trimmed = el.trim();
+      if (!Constants.UUIDRegex.test(trimmed)) {
+        throw Problem.user.unexpectedValue({
+          field: `query parameter '${queryParamName}'`,
+          value: trimmed,
+          reason: `is not interpretable as a UUID: ${el}`,
+        });
+      } else {
+        return trimmed;
+      }
+    });
+  }
+
+
   static queryParamToIntArray(queryParam, queryParamName) {
     return this.queryParamToArray(queryParam).map(el => {
       const inted = parseInt(el, 10);

lib/util/util.js

@@ -36,6 +36,8 @@ const truncateString = length => str => (str ? str.substring(0, length) : str);
 const sanitizeOdataIdentifier = (name) =>
   name.replace(/^([^a-z_])/i, '_$1').replace(/([^a-z0-9_]+)/gi, '_');
 
+const arrayHasElements = (fnarg) => (Array.isArray(fnarg) && fnarg.length);
+
 ////////////////////////////////////////
 // OBJECTS
 
@@ -88,7 +90,7 @@ const attachmentToDatasetName = (attachmentName) => attachmentName.replace(/\.cs
 
 module.exports = {
   noop, noargs, applyPipe,
-  isBlank, isPresent, blankStringToNull, truncateString, sanitizeOdataIdentifier,
+  isBlank, isPresent, blankStringToNull, truncateString, sanitizeOdataIdentifier, arrayHasElements,
   printPairs, omit, pickAll,
   base64ToUtf8, utf8ToBase64,
   construct, attachmentToDatasetName