Merge pull request #474 from helium/update/entity-invalidator

Update entity-invalidator to invalidate /v2/hotspots
helium · Nov 7, 2023 · 83b5afd · 83b5afd
2 parents 0a0a6ac + c955c97
commit 83b5afd
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 2 deletions.
diff --git a/packages/entity-invalidator/package.json b/packages/entity-invalidator/package.json
@@ -42,6 +42,7 @@
     "pg": "^8.9.0",
     "prom-client": "^14.2.0",
     "sequelize": "^6.28.0",
+    "uuid": "^9.0.1",
     "yargs": "^17.7.1"
   },
   "devDependencies": {

diff --git a/packages/entity-invalidator/src/env.ts b/packages/entity-invalidator/src/env.ts
@@ -3,6 +3,7 @@ export const LOOKBACK_HOURS = process.env.LOOKBACK_HOURS
   : 25;
 export const AWS_REGION = process.env.AWS_REGION;
 export const CLOUDFRONT_DISTRIBUTION = process.env.CLOUDFRONT_DISTRIBUTION!;
+export const DOMAIN = process.env.DOMAIN;
 
 // Check for required environment variables
 const requiredEnvVars = ["AWS_REGION", "CLOUDFRONT_DISTRIBUTION"];

diff --git a/packages/entity-invalidator/src/index.ts b/packages/entity-invalidator/src/index.ts
@@ -1,16 +1,111 @@
 import { decodeEntityKey } from "@helium/helium-entity-manager-sdk";
 import AWS from "aws-sdk";
 import { Op } from "sequelize";
+import { v4 as uuidv4 } from 'uuid';
 import { IotHotspotInfo, KeyToAsset, MobileHotspotInfo } from "./model";
 // @ts-ignore
-import { AWS_REGION, CLOUDFRONT_DISTRIBUTION, LOOKBACK_HOURS } from "./env";
+import { AWS_REGION, CLOUDFRONT_DISTRIBUTION, LOOKBACK_HOURS, DOMAIN } from "./env";
 
 async function run() {
   const date = new Date();
   date.setHours(date.getHours() - LOOKBACK_HOURS);
   AWS.config.update({ region: AWS_REGION });
   const cloudfront = new AWS.CloudFront();
 
+  // Invalidate metadata-service routes: 
+  // - /v2/hotspots/pagination-metadata?subnetwork=iot
+  // - /v2/hotspots/pagination-metadata?subnetwork=mobile
+  // Or /v2/hotspot* if there is an error
+  try {
+    const modelMap = {
+      'iot': IotHotspotInfo,
+      'mobile': MobileHotspotInfo,
+    }
+    const subnetworks = ['iot', 'mobile'];
+
+    // Fetch pagination data
+    const responsePromises = subnetworks.map((subnetwork) => {
+      return fetch(`https://${DOMAIN}/v2/hotspots/pagination-metadata?subnetwork=${subnetwork}`);
+    });
+    const jsonPromises = await Promise.all(responsePromises);
+    const paginationMetadata = await Promise.all(jsonPromises.map((response) => response.json()));
+    console.log("Fetched pagination metadata for subnetworks");
+    console.log(paginationMetadata);
+
+    // Fetch counts of newly added hotspots 
+    const totalCountPromises = subnetworks.map((subnetwork) => {
+      const whereClause = {
+        created_at: {
+          [Op.gte]: date,
+        },
+      };
+
+      return KeyToAsset.count({
+        where: whereClause,
+        include: [
+          {
+            model: modelMap[subnetwork],
+            required: true,
+          },
+        ],
+      });
+    });
+    const totalCounts = await Promise.all(totalCountPromises);
+    console.log("Fetched counts of newly added hotspots");
+    console.log(totalCounts);
+
+    // Prepare invalidation paths 
+    const paths: string[] = [];
+    totalCounts.forEach((count, i) => {
+      const subnetwork = subnetworks[i];
+      const countToPageSizeRatio = count / paginationMetadata[i].pageSize;
+      let numOfPagesToInvalidate = Math.ceil(countToPageSizeRatio);
+      while (numOfPagesToInvalidate >= 0) {
+        const page = paginationMetadata[i].totalPages - numOfPagesToInvalidate;
+        if (page > 0) {
+          const path = `/v2/hotspots?subnetwork=${subnetwork}&page=${page}`;
+          paths.push(path);
+        }
+        numOfPagesToInvalidate--;
+      }
+    });
+    console.log("Invalidation paths");
+    console.log(paths);
+
+    await cloudfront
+      .createInvalidation({
+        DistributionId: CLOUDFRONT_DISTRIBUTION,
+        InvalidationBatch: {
+          CallerReference: `${uuidv4()}`,
+          Paths: {
+            Quantity: paths.length,
+            Items: paths,
+          },
+        },
+      })
+      .promise();
+  } catch (err) {
+    console.error("Granular /v2/hotspots invalidation failed, resorting to full invalidation");
+    console.error(err);
+
+    await cloudfront
+      .createInvalidation({
+        DistributionId: CLOUDFRONT_DISTRIBUTION,
+        InvalidationBatch: {
+          CallerReference: `${uuidv4()}`,
+          Paths: {
+            Quantity: 1,
+            Items: ["/v2/hotspots*"],
+          },
+        },
+      })
+      .promise();
+  }
+
+  // Invalidate metadata-service routes:
+  // - /v2/hotspot/:keyToAssetKey
+  // - /v1/:keyToAssetKey 
+  // - /:eccCompact
   const limit = 10000;
   let lastId = null;
   let entities;
@@ -87,7 +182,7 @@ async function run() {
         .createInvalidation({
           DistributionId: CLOUDFRONT_DISTRIBUTION,
           InvalidationBatch: {
-            CallerReference: `${new Date().getTime()}`, // unique identifier for this invalidation batch
+            CallerReference: `${uuidv4()}`, // unique identifier for this invalidation batch
             Paths: {
               Quantity: paths.length,
               Items: paths,

diff --git a/packages/entity-invalidator/yarn.deploy.lock b/packages/entity-invalidator/yarn.deploy.lock
@@ -134,6 +134,7 @@ __metadata:
     ts-node: ^10.9.1
     ts-node-dev: ^2.0.0
     typescript: ^5.2.2
+    uuid: ^9.0.1
     yargs: ^17.7.1
   languageName: unknown
   linkType: soft
@@ -3187,6 +3188,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"uuid@npm:^9.0.1":
+  version: 9.0.1
+  resolution: "uuid@npm:9.0.1"
+  bin:
+    uuid: dist/bin/uuid
+  checksum: 39931f6da74e307f51c0fb463dc2462807531dc80760a9bff1e35af4316131b4fc3203d16da60ae33f07fdca5b56f3f1dd662da0c99fea9aaeab2004780cc5f4
+  languageName: node
+  linkType: hard
+
 "v8-compile-cache-lib@npm:^3.0.1":
   version: 3.0.1
   resolution: "v8-compile-cache-lib@npm:3.0.1"

diff --git a/yarn.lock b/yarn.lock
@@ -1178,6 +1178,7 @@ __metadata:
     ts-node: ^10.9.1
     ts-node-dev: ^2.0.0
     typescript: ^5.2.2
+    uuid: ^9.0.1
     yargs: ^17.7.1
   languageName: unknown
   linkType: soft
@@ -16140,6 +16141,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"uuid@npm:^9.0.1":
+  version: 9.0.1
+  resolution: "uuid@npm:9.0.1"
+  bin:
+    uuid: dist/bin/uuid
+  checksum: 39931f6da74e307f51c0fb463dc2462807531dc80760a9bff1e35af4316131b4fc3203d16da60ae33f07fdca5b56f3f1dd662da0c99fea9aaeab2004780cc5f4
+  languageName: node
+  linkType: hard
+
 "v8-compile-cache-lib@npm:^3.0.1":
   version: 3.0.1
   resolution: "v8-compile-cache-lib@npm:3.0.1"