Use sqlite cache of pantry data

.github/deno-to-node.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env -S pkgx +npm deno run --allow-env --allow-read --allow-write --allow-net --allow-run
 
-import { build, emptyDir } from "https://deno.land/x/dnt@0.38.1/mod.ts";
+import { build, emptyDir } from "https://deno.land/x/dnt@0.39.0/mod.ts";
 import SemVer from "../src/utils/semver.ts";
 
 await emptyDir("./dist");
@@ -33,7 +33,9 @@ await build({
   mappings: {
     "https://deno.land/x/[email protected]/src/index.ts": "is-what",
     "https://deno.land/x/[email protected]/mod.ts": "outdent",
-    "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts"
+    "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts",
+    "./src/hooks/useSyncCache.ts": "./src/hooks/useSyncCache.node.ts",
+    "./src/hooks/useSyncCache.test.ts": "./src/hooks/useCache.test.ts"  // no other easy way to skip the test
   },
   package: {
     name: "libpkgx",

.github/workflows/ci.yml

@@ -66,7 +66,7 @@ jobs:
         with:
           path: src
       - uses: denoland/setup-deno@v1
-      - run: deno run --no-config --unstable src/mod.ts
+      - run: deno run --no-config --unstable --allow-all src/mod.ts
 
   dnt:
     runs-on: ${{ matrix.os }}

deno.json

@@ -12,7 +12,7 @@
   },
   "pkgx": "deno^1.33.3 npm",
   "tasks": {
-    "test": "deno test --parallel --unstable --allow-env --allow-read --allow-net=dist.pkgx.dev,github.com,codeload.github.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe'",
+    "test": "deno test --parallel --unstable --allow-env --allow-read --allow-net=dist.pkgx.dev,github.com,codeload.github.com,objects.githubusercontent.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe' --allow-ffi",
     "typecheck": "deno check --unstable ./mod.ts",
     "dnt": ".github/deno-to-node.ts"
   },
@@ -26,7 +26,7 @@
   },
   "imports": {
     "is-what": "https://deno.land/x/[email protected]/src/index.ts",
-    "deno/": "https://deno.land/std@0.204.0/",
+    "deno/": "https://deno.land/std@0.196.0/",
     "outdent": "https://deno.land/x/[email protected]/mod.ts"
   }
 }

src/deps.ts

@@ -8,13 +8,13 @@ import * as outdent from "https://deno.land/x/[email protected]/mod.ts"
 export { outdent }
 
 // importing super specifically to reduce final npm bundle size
-import * as crypto from "https://deno.land/std@0.204.0/crypto/mod.ts"
-import { moveSync } from "https://deno.land/std@0.204.0/fs/move.ts"
-import { readLines } from "https://deno.land/std@0.204.0/io/read_lines.ts"
-import { writeAll } from "https://deno.land/std@0.204.0/streams/write_all.ts"
-import { parse as parseYaml } from "https://deno.land/std@0.204.0/yaml/parse.ts"
-import { SEP } from "https://deno.land/std@0.204.0/path/mod.ts"
-import { fromFileUrl } from "https://deno.land/std@0.204.0/path/from_file_url.ts"
+import * as crypto from "https://deno.land/std@0.196.0/crypto/mod.ts"
+import { moveSync } from "https://deno.land/std@0.196.0/fs/move.ts"
+import { readLines } from "https://deno.land/std@0.196.0/io/read_lines.ts"
+import { writeAll } from "https://deno.land/std@0.196.0/streams/write_all.ts"
+import { parse as parseYaml } from "https://deno.land/std@0.196.0/yaml/parse.ts"
+import { SEP } from "https://deno.land/std@0.196.0/path/mod.ts"
+import { fromFileUrl } from "https://deno.land/std@0.196.0/path/mod.ts"
 
 const streams = { writeAll }
 const io = { readLines }

src/hooks/usePantry.test.ts

@@ -85,3 +85,10 @@ Deno.test("validatePackageRequirement - number constraint", () => {
   const result = validatePackageRequirement("pkgx.sh/test", 1)
   assertEquals(result?.constraint.toString(), "^1")
 })
+
+Deno.test("find", async () => {
+  useTestConfig()
+  const foo = await usePantry().find("[email protected]")
+  assertEquals(foo.length, 1)
+  assertEquals(foo[0].project, "python.org")
+})

src/hooks/usePantry.ts

@@ -1,10 +1,12 @@
 import { is_what, PlainObject } from "../deps.ts"
 const { isNumber, isPlainObject, isString, isArray, isPrimitive, isBoolean } = is_what
 import { Package, Installation, PackageRequirement } from "../types.ts"
+import { provides as cache_provides, available as cache_available, runtime_env as cache_runtime_env, companions as cache_companions, dependencies as cache_dependencies } from "./useSyncCache.ts";
 import SemVer, * as semver from "../utils/semver.ts"
 import useMoustaches from "./useMoustaches.ts"
 import { PkgxError } from "../utils/error.ts"
 import { validate } from "../utils/misc.ts"
+import * as pkgutils from "../utils/pkg.ts"
 import useConfig from "./useConfig.ts"
 import host from "../utils/host.ts"
 import Path from "../utils/Path.ts"
@@ -45,6 +47,7 @@ export class PantryNotFoundError extends PantryError {
 
 export default function usePantry() {
   const prefix = useConfig().data.join("pantry/projects")
+  const is_cache_available = cache_available() && pantry_paths().length == 1
 
   async function* ls(): AsyncGenerator<LsEntry> {
     const seen = new Set()
@@ -78,11 +81,23 @@ export default function usePantry() {
       throw new PackageNotFoundError(project)
     })()
 
-    const companions = async () => parse_pkgs_node((await yaml())["companions"])
+    const companions = async () => {
+      if (is_cache_available) {
+        return await cache_companions(project) ?? parse_pkgs_node((await yaml())["companions"])
+      } else {
+        return parse_pkgs_node((await yaml())["companions"])
+      }
+    }
 
     const runtime_env = async (version: SemVer, deps: Installation[]) => {
-      const yml = await yaml()
-      const obj = validate.obj(yml["runtime"]?.["env"] ?? {})
+      const obj = await (async () => {
+        if (is_cache_available) {
+          const cached = await cache_runtime_env(project)
+          if (cached) return cached
+        }
+        const yml = await yaml()
+        return validate.obj(yml["runtime"]?.["env"] ?? {})
+      })()
       return expand_env_obj(obj, { project, version }, deps)
     }
 
@@ -94,7 +109,13 @@ export default function usePantry() {
       return platforms.includes(host().platform) ||platforms.includes(`${host().platform}/${host().arch}`)
     }
 
-    const drydeps = async () => parse_pkgs_node((await yaml()).dependencies)
+    const drydeps = async () => {
+      if (is_cache_available) {
+        return await cache_dependencies(project) ?? parse_pkgs_node((await yaml()).dependencies)
+      } else {
+        return parse_pkgs_node((await yaml()).dependencies)
+      }
+    }
 
     const provides = async () => {
       let node = (await yaml())["provides"]
@@ -164,6 +185,25 @@ export default function usePantry() {
   async function find(name: string) {
     type Foo = ReturnType<typeof project> & LsEntry
 
+    //lol FIXME
+    name = pkgutils.parse(name).project
+
+    if (prefix.join(name).isDirectory()) {
+      const foo = project(name)
+      return [{...foo, project: name }]
+    }
+
+    /// only use cache if PKGX_PANTRY_PATH is not set
+    if (is_cache_available) {
+      const cached = await cache_provides(name)
+      if (cached) {
+        return cached.map(x => ({
+          ...project(x),
+          project: x
+        }))
+      }
+    }
+
     name = name.toLowerCase()
 
     //TODO not very performant due to serial awaits
@@ -234,7 +274,8 @@ export default function usePantry() {
     parse_pkgs_node,
     expand_env_obj,
     missing,
-    neglected
+    neglected,
+    pantry_paths
   }
 
   function pantry_paths(): Path[] {

src/hooks/useSync.test.ts

@@ -1,44 +1,53 @@
+import specimen, { _internals } from "./useSync.ts"
 import { useTestConfig } from "./useTestConfig.ts"
+import * as mock from "deno/testing/mock.ts"
 import { assert } from "deno/assert/mod.ts"
 import usePantry from "./usePantry.ts"
-import useSync from "./useSync.ts"
 
 // NOTE actually syncs from github
 // TODO unit tests should not do actual network calls, instead make an implementation suite
 
 Deno.test("useSync", async runner => {
-  await runner.step("w/o git", async () => {
-    const conf = useTestConfig({})
-    usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
-    assert(conf.git === undefined)
-    await test()
-  })
-
-  await runner.step({
-    name: "w/git",
-    ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"),
-    async fn() {
-      const conf = useTestConfig({ PATH: "/usr/bin" })
+  const stub = mock.stub(_internals, "cache", async () => {})
+
+  try {
+    await runner.step("w/o git", async () => {
+      const conf = useTestConfig({})
       usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
-      assert(conf.git !== undefined)
+      assert(conf.git === undefined)
       await test()
+    })
 
-      // test the “already cloned” code-path
-      await useSync()
-    }
-  })
-
-  async function test() {
-    let errord = false
-    try {
-      await usePantry().project("gnu.org/gcc").available()
-    } catch {
-      errord = true
-    }
-    assert(errord, `should be no pantry but there is! ${usePantry().prefix}`)
+    await runner.step({
+      name: "w/git",
+      ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"),
+      async fn() {
+        const conf = useTestConfig({ PATH: "/usr/bin" })
+        usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
+        assert(conf.git !== undefined)
+        await test()
+
+        // test the “already cloned” code-path
+        await specimen()
+      }
+    })
 
-    await useSync()
+    async function test() {
+      let errord = false
+      try {
+        await usePantry().project("gnu.org/gcc").available()
+      } catch {
+        errord = true
+      }
+      assert(errord, `should be no pantry but there is! ${usePantry().prefix}`)
 
-    assert(await usePantry().project("gnu.org/gcc").available())
+      await specimen()
+
+      assert(await usePantry().project("gnu.org/gcc").available())
+    }
+
+  } finally {
+    stub.restore()
   }
+
 })

src/hooks/useSync.ts

@@ -7,12 +7,14 @@ import useDownload from "./useDownload.ts"
 import usePantry from "./usePantry.ts"
 import useConfig from "./useConfig.ts"
 import Path from "../utils/Path.ts"
+import useSyncCache from "./useSyncCache.ts";
 
 //FIXME tar is fetched from PATH :/ we want control
 //FIXME run in general is not controllable since it delegates to the shell
 
 interface Logger {
   syncing(path: Path): void
+  caching(path: Path): void
   syncd(path: Path): void
 }
 
@@ -23,6 +25,27 @@ export default async function(logger?: Logger) {
 
   const unflock = await flock(pantry_dir.mkdir('p'))
 
+  try {
+    await _internals.sync(pantry_dir)
+    try {
+      logger?.caching(pantry_dir)
+      await _internals.cache()
+    } catch (err) {
+      console.warn("failed to cache pantry")
+      console.error(err)
+    }
+  } finally {
+    await unflock()
+  }
+
+  logger?.syncd(pantry_dir)
+}
+
+export const _internals = {
+  sync, cache: useSyncCache
+}
+
+async function sync(pantry_dir: Path) {
   try {
     //TODO if there was already a lock, just wait on it, don’t do the following stuff
 
@@ -55,11 +78,7 @@ export default async function(logger?: Logger) {
 
     proc.close()
 
-  } finally {
-    await unflock()
   }
-
-  logger?.syncd(pantry_dir)
 }
 
 //////////////////////// utils

src/hooks/useSyncCache.node.ts

@@ -0,0 +1,31 @@
+// the sqlite lib we use only works in deno
+
+import { PackageRequirement } from "../../mod.ts";
+
+export default async function()
+{}
+
+export function provides(_program: string): string[] {
+  throw new Error()
+}
+
+export function dependencies(_project: string): PackageRequirement[] {
+  throw new Error()
+}
+
+export function completion(_prefix: string): string[] {
+  throw new Error()
+}
+
+/// is the cache available?
+export function available(): boolean {
+  return false
+}
+
+export function companions(_project: string): PackageRequirement[] {
+  throw new Error()
+}
+
+export function runtime_env(_project: string): Record<string, string> {
+  throw new Error()
+}

src/hooks/useSyncCache.test.ts

@@ -0,0 +1,26 @@
+import specimen, { provides, dependencies, available, runtime_env, completion, companions  } from "./useSyncCache.ts"
+import { useTestConfig } from "./useTestConfig.ts"
+import { assert, assertEquals } from "deno/assert/mod.ts"
+import { _internals } from "./useSync.ts"
+import usePantry from "./usePantry.ts"
+
+// NOTE actually syncs from github
+// TODO unit tests should not do actual network calls, instead make an implementation suite
+
+Deno.test({
+  name: "useSyncCache",
+  ignore: Deno.build.os == 'windows',
+  async fn() {
+    useTestConfig()
+    await _internals.sync(usePantry().prefix.parent())
+    await specimen()
+
+    //TODO test better
+    assert(available())
+    assertEquals((await provides('node'))?.[0], 'nodejs.org')
+    // assertEquals((await dependencies('nodejs.org'))?.length, 3)
+    assert(new Set(await completion('nod')).has("node"))
+    assertEquals((await companions("nodejs.org"))?.[0]?.project, "npmjs.com")
+    assert((await runtime_env("numpy.org"))?.["PYTHONPATH"])
+  }
+})