Backend usage metrics (offline entities) for 2024.2 (#1198)

* Start of offline entity analytics queries

* Counting submission.reprocess events

* Submission/entity wait time

* Updating interrupted branch count query

* Tests trying to show submission processing delays

* New overall entity time query

* Add new values to output and update form version

* replace approval required request

config/default.json

@@ -30,7 +30,7 @@
       "analytics": {
         "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/forms/odk-analytics/submissions",
         "formId": "odk-analytics",
-        "version": "v2024.1.0_1"
+        "version": "v2024.2.0_1"
       },
       "s3blobStore": {}
     }

lib/data/analytics.js

@@ -37,7 +37,12 @@ const metricsTemplate = {
     "num_client_audit_rows": 0,
     "num_audits_failed": 0,
     "num_audits_failed5": 0,
-    "num_audits_unprocessed": 0
+    "num_audits_unprocessed": 0,
+    "num_offline_entity_branches": 0,
+    "num_offline_entity_interrupted_branches": 0,
+    "num_offline_entity_submissions_reprocessed": 0,
+    "max_entity_submission_delay": 0,
+    "avg_entity_submission_delay": 0
   },
   "projects": [
     {

lib/model/query/analytics.js

@@ -394,7 +394,6 @@ group by f."projectId"`);
 
 
 // Datasets
-/* eslint-disable no-tabs */
 const getDatasets = () => ({ all }) => all(sql`
 SELECT 
   ds.id, ds."projectId", COUNT(DISTINCT p.id) num_properties, COUNT(DISTINCT e.id) num_entities_total,
@@ -462,7 +461,6 @@ FROM datasets ds
 WHERE ds."publishedAt" IS NOT NULL
 GROUP BY ds.id, ds."projectId"
 `);
-/* eslint-enable no-tabs */
 
 const getDatasetEvents = () => ({ all }) => all(sql`
 SELECT
@@ -477,6 +475,84 @@ WHERE audits.action = 'entity.bulk.create'
 GROUP BY ds.id, ds."projectId"
 `);
 
+
+// Offline entities
+
+// Number of offline branches involving more than one update
+// Updates from offline-enabled Collect will include branchId so it is not enough
+// to count that but we can look at trunkVersion and branchBaseVersion to find
+// versions that had a true (multi-step) offlne operation.
+const countOfflineBranches = () => ({ oneFirst }) => oneFirst(sql`
+SELECT COUNT(DISTINCT "branchId")
+FROM entity_defs 
+WHERE "branchId" IS NOT NULL AND ("trunkVersion" IS NULL OR "branchBaseVersion" > "trunkVersion")
+`);
+
+// Look up offline branches that have another branchId
+// interrupting them, E.g. abc, abc, xyz, abc
+const countInterruptedBranches = () => ({ oneFirst }) => oneFirst(sql`
+WITH sortedRows AS (
+    SELECT 
+        "entityId",
+        "version",
+        "branchId",
+        LAG("branchId") OVER (PARTITION BY "entityId" ORDER BY "version") AS "prevBranchId"
+    FROM entity_defs
+),
+distinctRuns AS (
+    SELECT 
+        "entityId",
+        "branchId"
+    FROM sortedRows
+    WHERE "version" = 1 OR "branchId" IS DISTINCT FROM "prevBranchId" -- Keep first row and changes
+),
+duplicateRuns AS (
+    SELECT 
+        "entityId",
+        "branchId",
+        COUNT(*) AS runCount
+    FROM distinctRuns
+    WHERE "branchId" IS NOT NULL
+    GROUP BY "entityId", "branchId"
+    HAVING COUNT(*) > 1 -- Selects branchIds that occur more than once
+)
+SELECT COUNT(*)
+FROM duplicateRuns;
+`);
+
+// Number of submissions temporarily held in backlog but were automatically
+// removed from backlog when preceeding submission came in
+const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql`
+  SELECT COUNT(*)
+  FROM audits
+  WHERE "action" = 'submission.reprocess'
+`);
+
+// Measure how much time entities whose source is a submission.create
+// event take to process to look for a processing lag. We look at the
+// submission create loggedAt timestamp (when sub was created) to when
+// the event was processed, which will be after the entity version was
+// created.
+const measureEntityProcessingTime = () => ({ one }) => one(sql`
+SELECT
+  MAX("processed"-"loggedAt") as max_wait,
+  AVG("processed"-"loggedAt") as avg_wait
+FROM entity_def_sources
+JOIN audits ON audits.id = "auditId"
+WHERE action = 'submission.create'
+`);
+
+const measureElapsedEntityTime = () => ({ one }) => one(sql`
+SELECT
+  MAX(ed."createdAt" - sd."createdAt") as max_wait,
+  AVG(ed."createdAt" - sd."createdAt") as avg_wait
+FROM entity_defs as ed
+JOIN entity_def_sources as eds
+  ON ed."sourceId" = eds."id"
+JOIN submission_defs as sd
+  ON eds."submissionDefId" = sd.id;
+`);
+
 // Other
 const getProjectsWithDescriptions = () => ({ all }) => all(sql`
 select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`);
@@ -662,10 +738,15 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([
   Analytics.countClientAuditAttachments(),
   Analytics.countClientAuditProcessingFailed(),
   Analytics.countClientAuditRows(),
+  Analytics.countOfflineBranches(),
+  Analytics.countInterruptedBranches(),
+  Analytics.countSubmissionReprocess(),
+  Analytics.measureEntityProcessingTime(),
   Analytics.projectMetrics()
 ]).then(([db, encrypt, bigForm, admins, audits,
   archived, managers, viewers, collectors,
   caAttachments, caFailures, caRows,
+  oeBranches, oeInterruptedBranches, oeSubReprocess, oeProcessingTime,
   projMetrics]) => {
   const metrics = clone(metricsTemplate);
   // system
@@ -700,6 +781,13 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([
   metrics.system.num_audits_unprocessed = audits.unprocessed;
   metrics.system.sso_enabled = oidc.isEnabled() ? 1 : 0;
 
+  // 2024.2.0 offline entity metrics
+  metrics.system.num_offline_entity_branches = oeBranches;
+  metrics.system.num_offline_entity_interrupted_branches = oeInterruptedBranches;
+  metrics.system.num_offline_entity_submissions_reprocessed = oeSubReprocess;
+  metrics.system.max_entity_submission_delay = oeProcessingTime.max_wait;
+  metrics.system.avg_entity_submission_delay = oeProcessingTime.avg_wait;
+
   return metrics;
 }));
 
@@ -750,5 +838,10 @@ module.exports = {
   projectMetrics,
   getLatestAudit,
   getDatasets,
-  getDatasetEvents
+  getDatasetEvents,
+  countOfflineBranches,
+  countInterruptedBranches,
+  countSubmissionReprocess,
+  measureEntityProcessingTime,
+  measureElapsedEntityTime
 };