Merge pull request #37 from desci-labs/m0ar/export-deterministic-runtime

Deploying composites now build deterministic runtime files

.github/workflows/publish.yml

@@ -15,6 +15,9 @@ jobs:
           check-latest: false
           registry-url: 'https://registry.npmjs.org'
       - run: npm ci
+      - run: npm --workspace packages/composedb publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
       - run: npm --workspace packages/lib publish --access public
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

packages/composedb/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@desci-labs/desci-codex-composedb",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "ComposeDB models and scripts for Codex",
   "license": "MIT",
   "author": "Edvard Hübinette",

packages/composedb/scripts/composites.ts

@@ -1,4 +1,4 @@
-import { readFileSync } from "fs";
+import { readFileSync, writeFileSync } from "fs";
 import { CeramicClient } from "@ceramicnetwork/http-client";
 import {
   createComposite,
@@ -14,6 +14,8 @@ import { fromString } from "uint8arrays/from-string";
 import { Ora } from "ora";
 
 const ceramic = new CeramicClient("http://localhost:7007");
+const ENCODED_PATH = "./src/__generated__/definition.json";
+const ENCODED_RUNTIME_PATH = "./src/__generated__/definition.js";
 
 type Models = {
   profile?: string;
@@ -174,18 +176,19 @@ export const writeComposite = async (seed: string, spinner?: Ora) => {
     annotationComposite,
   ]);
 
-  await writeEncodedComposite(composite, "./src/__generated__/definition.json");
+  await writeEncodedComposite(composite, ENCODED_PATH);
   spinner.info("creating composite for runtime usage");
   await writeEncodedCompositeRuntime(
     ceramic,
-    "./src/__generated__/definition.json",
-    "./src/__generated__/definition.js",
+    ENCODED_PATH,
+    ENCODED_RUNTIME_PATH,
   );
+
+  // Fix non-determinism due to arbitrarily sorted keys in files
+  await orderCompositeFileKeys();
+
   spinner.info("deploying composite");
-  const deployComposite = await readEncodedComposite(
-    ceramic,
-    "./src/__generated__/definition.json",
-  );
+  const deployComposite = await readEncodedComposite(ceramic, ENCODED_PATH);
 
   await deployComposite.startIndexingOn(ceramic);
   spinner.succeed("composite deployed & ready for use");
@@ -204,6 +207,41 @@ const authenticateAdmin = async (seed: string): Promise<void> => {
   await ceramic.setDID(did);
 };
 
+/**
+ * Repeated runs yield diffs in generated composite definition files, even
+ * if they are semantically the same, because the keys aren't serialized
+ * in order. This function fixes that, making it clear in git when they
+ * actually have changed.
+ */
+const orderCompositeFileKeys = async () => {
+  /* eslint-disable @typescript-eslint/no-explicit-any*/
+
+  /**
+   * Recursively order object keys to get deterministic serialization
+   * https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490
+   */
+  const orderedReplacer = (_key: any, value: any) =>
+    value instanceof Object && !(value instanceof Array)
+      ? Object.keys(value)
+          .sort()
+          .reduce((sorted, key) => {
+            sorted[key] = value[key];
+            return sorted;
+          }, {} as any)
+      : value;
+
+  const encoded = JSON.parse(readFileSync(ENCODED_PATH, { encoding: "ascii" }));
+  writeFileSync(ENCODED_PATH, JSON.stringify(encoded, orderedReplacer));
+
+  const { definition } = await import(
+    process.cwd() + "/" + ENCODED_RUNTIME_PATH
+  );
+  const encoded_runtime_ordered = `
+export const definition = ${JSON.stringify(definition, orderedReplacer)}
+`;
+  writeFileSync(ENCODED_RUNTIME_PATH, encoded_runtime_ordered);
+};
+
 const runAsScript =
   process.argv[0].includes("/bin/node") &&
   process.argv[1].includes("scripts/composites.ts");

packages/lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@desci-labs/desci-codex-lib",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Codex interaction primitives",
   "license": "MIT",
   "author": "Edvard Hübinette",
@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "@composedb/client": "^0.6.0",
+    "@desci-labs/desci-codex-composedb": "^1.0.1",
     "dids": "^4.0.4",
     "gql-query-builder": "^3.8.0",
     "graphql": "^16.8.0",

packages/lib/src/clients.ts

@@ -7,7 +7,7 @@ import {
   CeramicClient,
   type CeramicClientConfig,
 } from "@ceramicnetwork/http-client";
-import type { Optional } from "./types.js";
+import { definition } from "@desci-labs/desci-codex-composedb/src/__generated__/definition.js";
 
 const DEFAULT_LOCAL_CERAMIC = "http://localhost:7007";
 
@@ -35,9 +35,7 @@ export const newCeramicClient = (
   return new CeramicClient(endpoint ?? DEFAULT_LOCAL_CERAMIC, config);
 };
 
-export const newComposeClient = (
-  params: Optional<ComposeClientParams, "ceramic">,
-) => {
+export const newComposeClient = (params: Partial<ComposeClientParams>) => {
   if (!params.ceramic) {
     console.log(
       "[codex] ceramic client not provided; defaulting to",
@@ -47,6 +45,7 @@ export const newComposeClient = (
 
   return new ComposeClient({
     ceramic: DEFAULT_LOCAL_CERAMIC,
+    definition,
     // Let passed config overwrite ceramic, if present
     ...params,
   });