Add usage metrics for central release 2025.3 (#1662)

* Added num_entity_bulk_deletes usage metric

* Counting max geo data in cache in a single form: max_geo_per_form

* Updating form version

* Add num_datasets_with_geometry to count system-wide

* Count enitites with geometry per dataset: num_entities_with_geometry

* Queries to count entity forms, specified by action and repeat

Author: ktuite

config/default.json

@@ -30,7 +30,7 @@
       "analytics": {
         "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/forms/odk-analytics/submissions",
         "formId": "odk-analytics",
-        "version": "v2025.2.0_1"
+        "version": "v2025.3.0_1"
       },
       "s3blobStore": {}
     },

lib/data/analytics.js

@@ -21,6 +21,7 @@ const metricsTemplate = {
       "total": 0
     },
     "num_questions_biggest_form": {},
+    "max_geo_per_form": 0,
     "num_audit_log_entries": {
       "recent": 0,
       "total": 0
@@ -50,7 +51,12 @@ const metricsTemplate = {
     "num_xml_only_form_defs": 0,
     "num_blob_files": 0,
     "num_blob_files_on_s3": 0,
-    "num_reset_failed_to_pending_count": 0
+    "num_reset_failed_to_pending_count": 0,
+    "num_entity_bulk_deletes": {
+      "recent": 0,
+      "total": 0
+    },
+    "num_datasets_with_geometry": 0
   },
   "projects": [
     {
@@ -115,7 +121,12 @@ const metricsTemplate = {
         "num_closed_forms": {
           "recent": 0,
           "total": 0
-        }
+        },
+        "num_entity_create_forms": 0,
+        "num_repeat_entity_create_forms": 0,
+        "num_entity_update_forms": 0,
+        "num_repeat_entity_update_forms": 0,
+        "num_entity_create_update_forms": 0,
       },
       "submissions": {
         "num_submissions_received": {
@@ -197,7 +208,11 @@ const metricsTemplate = {
           total: 0,
           recent: 0
         },
-        "biggest_bulk_upload": 0
+        "biggest_bulk_upload": 0,
+        "num_entities_with_geometry": {
+          total: 0,
+          recent: 0
+        }
       }]
     }
   ]

lib/model/query/analytics.js

@@ -9,7 +9,7 @@
 
 const config = require('config');
 const { sql } = require('slonik');
-const { clone } = require('ramda');
+const { clone, omit } = require('ramda');
 const { runSequentially } = require('../../util/promise');
 const { metricsTemplate } = require('../../data/analytics');
 const oidc = require('../../util/oidc');
@@ -344,6 +344,33 @@ on forms."xmlFormId" = deleted_form_ids."xml_form_id" and
 where forms."deletedAt" is null
 group by forms."projectId"`);
 
+const maxGeoPerForm = () => ({ oneFirst }) => oneFirst(sql`
+  SELECT MAX(geo_count)
+  FROM (
+    SELECT COUNT(*) as geo_count, fd."formId"
+    FROM submission_field_extract_geo_cache AS sc
+    JOIN submission_defs AS sd
+      ON sd.id = sc."submission_def_id"
+    JOIN form_defs AS fd
+      ON sd."formDefId" = fd.id
+    GROUP BY fd."formId", sc.fieldhash
+  ) as geo_counts`);
+
+const countFormsCreateUpdateEntitiesFromRepeats = () => ({ all }) => all(sql`
+SELECT
+  COUNT(DISTINCT CASE WHEN dfd.actions ? 'create' THEN f.id END) AS num_entity_create_forms,
+  COUNT(DISTINCT CASE WHEN dfd.actions ? 'create' AND dfd."isRepeat" THEN f.id END) AS num_repeat_entity_create_forms,
+  COUNT(DISTINCT CASE WHEN dfd.actions ? 'update' THEN f.id END) AS num_entity_update_forms,
+  COUNT(DISTINCT CASE WHEN dfd.actions ? 'update' AND dfd."isRepeat" THEN f.id END) AS num_repeat_entity_update_forms,
+  COUNT(DISTINCT CASE WHEN dfd.actions ? 'create' AND dfd.actions ? 'update' THEN f.id END) AS num_entity_create_update_forms,
+  f."projectId"
+FROM forms AS f
+  JOIN form_defs AS fd ON f."currentDefId" = fd."id"
+  JOIN dataset_form_defs AS dfd ON dfd."formDefId" = fd.id
+WHERE f."deletedAt" IS NULL
+GROUP BY f."projectId"
+ORDER BY f."projectId"`);
+
 // Submissions
 const countSubmissions = () => ({ all }) => all(sql`
 select f."projectId", count(s.id) as total,
@@ -540,7 +567,7 @@ const getDatasets = () => ({ all }) => all(sql`
 
 const getDatasetEvents = () => ({ all }) => all(sql`
 SELECT
-  ds.id, ds."projectId",
+  ds.id "datasetId", ds."projectId",
   COUNT (*) num_bulk_create_events_total,
   SUM (CASE WHEN audits."loggedAt" >= ${_cutoffDate} THEN 1 ELSE 0 END) num_bulk_create_events_recent,
   MAX (CAST(sources."details"->'count' AS integer)) AS biggest_bulk_upload
@@ -553,7 +580,7 @@ GROUP BY ds.id, ds."projectId"
 
 const getDatasetProperties = () => ({ all }) => all(sql`
 SELECT
-  ds.id, ds."projectId", COUNT(DISTINCT p.id) num_properties
+  ds.id "datasetId", ds."projectId", COUNT(DISTINCT p.id) num_properties
 FROM datasets ds
   LEFT JOIN ds_properties p ON p."datasetId" = ds.id AND p."publishedAt" IS NOT NULL
 WHERE ds."publishedAt" IS NOT NULL
@@ -661,6 +688,36 @@ const countOwnerOnlyDatasets = () => ({ oneFirst }) => oneFirst(sql`
   SELECT COUNT(1) FROM datasets where "ownerOnly" = true;
 `);
 
+const countDatasetsWithGeometry = () => ({ oneFirst }) => oneFirst(sql`
+  SELECT COUNT(DISTINCT "datasetId") FROM ds_properties WHERE name = 'geometry';
+`);
+
+const countEntitiesWithGeometry = () => ({ all }) => all(sql`
+SELECT
+  COUNT(*) AS total,
+  COUNT(CASE WHEN e."createdAt" >= ${_cutoffDate}
+    THEN 1
+    ELSE null
+  END) AS recent,
+  ds."projectId",
+  ds.id as "datasetId"
+FROM entities AS e
+  JOIN entity_defs AS ed ON ed."entityId" = e.id AND ed.current = true
+  JOIN datasets AS ds ON ds.id = e."datasetId"
+WHERE ed.data ? 'geometry'
+  AND e."deletedAt" IS NULL
+  AND ds."publishedAt" IS NOT NULL
+GROUP BY ds.id, ds."projectId"`);
+
+const countEntityBulkDeletes = () => ({ one }) => one(sql`
+SELECT
+  count(*) AS total,
+  count(CASE WHEN "loggedAt" >= ${_cutoffDate}
+    THEN 1
+    ELSE null
+  END) AS recent
+FROM audits WHERE action = 'entity.bulk.delete'`);
+
 // Other
 const getProjectsWithDescriptions = () => ({ all }) => all(sql`
 select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`);
@@ -676,6 +733,7 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.countFormsInStates,
   Analytics.countFormsWebformsEnabled,
   Analytics.countReusedFormIds,
+  Analytics.countFormsCreateUpdateEntitiesFromRepeats,
   Analytics.countSubmissions,
   Analytics.countSubmissionReviewStates,
   Analytics.countSubmissionsEdited,
@@ -684,11 +742,12 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.getProjectsWithDescriptions,
   Analytics.getDatasets,
   Analytics.getDatasetEvents,
-  Analytics.getDatasetProperties
+  Analytics.getDatasetProperties,
+  Analytics.countEntitiesWithGeometry,
 ]).then(([ userRoles, appUsers, deviceIds, pubLinks,
-  forms, formGeoRepeats, formsEncrypt, formStates, webforms, reusedIds,
+  forms, formGeoRepeats, formsEncrypt, formStates, webforms, reusedIds, entityForms,
   subs, subStates, subEdited, subComments, subUsers,
-  projWithDesc, datasets, datasetEvents, datasetProperties ]) => {
+  projWithDesc, datasets, datasetEvents, datasetProperties, entitiesWithGeometry ]) => {
   const projects = {};
 
   // users
@@ -762,6 +821,11 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
     project.forms.num_reused_form_ids = row.total;
   }
 
+  for (const row of entityForms) {
+    const project = _getProject(projects, row.projectId);
+    Object.assign(project.forms, omit(['projectId'], row));
+  }
+
   // submissions
   for (const row of subs) {
     const project = _getProject(projects, row.projectId);
@@ -798,21 +862,28 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
     project.submissions.num_submissions_from_web_users = { total: row.web_user_total, recent: row.web_user_recent };
   }
 
+
+  // Helper function
+  const findDataset = (metric, dataset, defaultValue) =>
+    metric.find(d => (d.datasetId === dataset.id)) || defaultValue;
+
   // datasets
   for (const row of datasets) {
     const project = _getProject(projects, row.projectId);
 
     // Additional dataset metrics are returned in a separate query. Look up the correct dataset/project row.
-    const eventsRow = datasetEvents.find(d => (d.projectId === row.projectId && d.id === row.id)) ||
-      { num_bulk_create_events_total: 0, num_bulk_create_events_recent: 0, biggest_bulk_upload: 0 };
+    const eventsRow = findDataset(datasetEvents, row, { num_bulk_create_events_total: 0, num_bulk_create_events_recent: 0, biggest_bulk_upload: 0 });
 
     // Properties row
-    const propertiesRow = datasetProperties.find(d => (d.projectId === row.projectId && d.id === row.id)) ||
-    { num_properties: 0 };
+    const propertiesRow = findDataset(datasetProperties, row, { num_properties: 0 });
+
+    // Entities with geometry
+    const entitiesWithGeometryRow = findDataset(entitiesWithGeometry, row, { total: 0, recent: 0 });
 
     project.datasets.push({
       id: row.id,
       num_properties: propertiesRow.num_properties,
+      num_entities_with_geometry: { total: entitiesWithGeometryRow.total, recent: entitiesWithGeometryRow.recent },
       num_creation_forms: row.num_creation_forms,
       num_followup_forms: row.num_followup_forms,
       num_entities: { total: row.num_entities_total, recent: row.num_entities_recent },
@@ -837,6 +908,7 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
     });
   }
 
+
   // other
   for (const row of projWithDesc) {
     const project = _getProject(projects, row.projectId);
@@ -852,6 +924,7 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.databaseSize,
   Analytics.encryptedProjects,
   Analytics.biggestForm,
+  Analytics.maxGeoPerForm,
   Analytics.countAdmins,
   Analytics.auditLogs,
   Analytics.archivedProjects,
@@ -870,12 +943,16 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.measureEntityProcessingTime,
   Analytics.measureMaxEntityBranchTime,
   Analytics.countOwnerOnlyDatasets,
+  Analytics.countDatasetsWithGeometry,
+  Analytics.countEntityBulkDeletes,
   Analytics.projectMetrics
-]).then(([db, encrypt, bigForm, admins, audits,
+]).then(([db, encrypt, bigForm, maxGeo, admins, audits,
   archived, managers, viewers, collectors,
   caAttachments, caFailures, caRows, xmlDefs, blobFiles, resetFailedToPending,
   oeBranches, oeInterruptedBranches, oeBacklogEvents, oeProcessingTime, oeBranchTime,
   ownerOnlyDatasets,
+  datasetsWithGeometry,
+  entityBulkDeletes,
   projMetrics]) => {
   const metrics = clone(metricsTemplate);
   // system
@@ -891,6 +968,7 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([
   for (const [key, value] of Object.entries(encrypt))
     metrics.system.num_projects_encryption[key] = value;
   metrics.system.num_questions_biggest_form = bigForm;
+  metrics.system.max_geo_per_form = maxGeo || 0;
   for (const [key, value] of Object.entries(admins))
     metrics.system.num_admins[key] = value;
 
@@ -930,6 +1008,12 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([
   // 2025.2.0 owner only entity lists/datasets
   metrics.system.num_owner_only_datasets = ownerOnlyDatasets;
 
+  // 2025.3.0 entity bulk delete audit logs
+  metrics.system.num_entity_bulk_deletes = entityBulkDeletes;
+
+  // 2025.3.0 datasets with geometry property
+  metrics.system.num_datasets_with_geometry = datasetsWithGeometry || 0;
+
   return metrics;
 }));
 
@@ -968,6 +1052,8 @@ module.exports = {
   countFormFieldTypes,
   countFormsInStates,
   countFormsWebformsEnabled,
+  maxGeoPerForm,
+  countFormsCreateUpdateEntitiesFromRepeats,
   countPublicLinks,
   countReusedFormIds,
   countSubmissions,
@@ -993,5 +1079,8 @@ module.exports = {
   measureEntityProcessingTime,
   measureElapsedEntityTime,
   measureMaxEntityBranchTime,
-  countOwnerOnlyDatasets
+  countOwnerOnlyDatasets,
+  countDatasetsWithGeometry,
+  countEntitiesWithGeometry,
+  countEntityBulkDeletes
 };