From c9f7161ca37205d1249b4fd596987e78dc84b90b Mon Sep 17 00:00:00 2001 From: Max Howell Date: Mon, 22 Jan 2024 15:39:06 -0500 Subject: [PATCH] Use sqlite cache of pantry data --- .github/deno-to-node.ts | 6 +- .github/workflows/ci.yml | 2 +- deno.json | 4 +- src/deps.ts | 14 +-- src/hooks/usePantry.test.ts | 7 ++ src/hooks/usePantry.ts | 51 +++++++++- src/hooks/useSync.test.ts | 67 ++++++------ src/hooks/useSync.ts | 27 ++++- src/hooks/useSyncCache.node.ts | 31 ++++++ src/hooks/useSyncCache.test.ts | 26 +++++ src/hooks/useSyncCache.ts | 181 +++++++++++++++++++++++++++++++++ src/hooks/useTestConfig.ts | 2 +- src/plumbing/which.ts | 20 +++- 13 files changed, 384 insertions(+), 54 deletions(-) create mode 100644 src/hooks/useSyncCache.node.ts create mode 100644 src/hooks/useSyncCache.test.ts create mode 100644 src/hooks/useSyncCache.ts diff --git a/.github/deno-to-node.ts b/.github/deno-to-node.ts index 7e62ff1..791aa5e 100755 --- a/.github/deno-to-node.ts +++ b/.github/deno-to-node.ts @@ -1,6 +1,6 @@ #!/usr/bin/env -S pkgx +npm deno run --allow-env --allow-read --allow-write --allow-net --allow-run -import { build, emptyDir } from "https://deno.land/x/dnt@0.38.1/mod.ts"; +import { build, emptyDir } from "https://deno.land/x/dnt@0.39.0/mod.ts"; import SemVer from "../src/utils/semver.ts"; await emptyDir("./dist"); @@ -33,7 +33,9 @@ await build({ mappings: { "https://deno.land/x/is_what@v4.1.15/src/index.ts": "is-what", "https://deno.land/x/outdent@v0.8.0/mod.ts": "outdent", - "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts" + "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts", + "./src/hooks/useSyncCache.ts": "./src/hooks/useSyncCache.node.ts", + "./src/hooks/useSyncCache.test.ts": "./src/hooks/useCache.test.ts" // no other easy way to skip the test }, package: { name: "libpkgx", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82b06fe..926f901 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: with: path: src - uses: denoland/setup-deno@v1 - - run: deno run --no-config --unstable src/mod.ts + - run: deno run --no-config --unstable --allow-all src/mod.ts dnt: runs-on: ${{ matrix.os }} diff --git a/deno.json b/deno.json index 0d75dbe..1a6a167 100644 --- a/deno.json +++ b/deno.json @@ -12,7 +12,7 @@ }, "pkgx": "deno^1.33.3 npm", "tasks": { - "test": "deno test --parallel --unstable --allow-env --allow-read --allow-net=dist.pkgx.dev,github.com,codeload.github.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe'", + "test": "deno test --parallel --unstable --allow-env --allow-read --allow-net=dist.pkgx.dev,github.com,codeload.github.com,objects.githubusercontent.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe' --allow-ffi", "typecheck": "deno check --unstable ./mod.ts", "dnt": ".github/deno-to-node.ts" }, @@ -26,7 +26,7 @@ }, "imports": { "is-what": "https://deno.land/x/is_what@v4.1.15/src/index.ts", - "deno/": "https://deno.land/std@0.204.0/", + "deno/": "https://deno.land/std@0.196.0/", "outdent": "https://deno.land/x/outdent@v0.8.0/mod.ts" } } diff --git a/src/deps.ts b/src/deps.ts index 08c54fa..7ec2015 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -8,13 +8,13 @@ import * as outdent from "https://deno.land/x/outdent@v0.8.0/mod.ts" export { outdent } // importing super specifically to reduce final npm bundle size -import * as crypto from "https://deno.land/std@0.204.0/crypto/mod.ts" -import { moveSync } from "https://deno.land/std@0.204.0/fs/move.ts" -import { readLines } from "https://deno.land/std@0.204.0/io/read_lines.ts" -import { writeAll } from "https://deno.land/std@0.204.0/streams/write_all.ts" -import { parse as parseYaml } from "https://deno.land/std@0.204.0/yaml/parse.ts" -import { SEP } from "https://deno.land/std@0.204.0/path/mod.ts" -import { fromFileUrl } from "https://deno.land/std@0.204.0/path/from_file_url.ts" +import * as crypto from "https://deno.land/std@0.196.0/crypto/mod.ts" +import { moveSync } from "https://deno.land/std@0.196.0/fs/move.ts" +import { readLines } from "https://deno.land/std@0.196.0/io/read_lines.ts" +import { writeAll } from "https://deno.land/std@0.196.0/streams/write_all.ts" +import { parse as parseYaml } from "https://deno.land/std@0.196.0/yaml/parse.ts" +import { SEP } from "https://deno.land/std@0.196.0/path/mod.ts" +import { fromFileUrl } from "https://deno.land/std@0.196.0/path/mod.ts" const streams = { writeAll } const io = { readLines } diff --git a/src/hooks/usePantry.test.ts b/src/hooks/usePantry.test.ts index 99609d0..a49ef59 100644 --- a/src/hooks/usePantry.test.ts +++ b/src/hooks/usePantry.test.ts @@ -85,3 +85,10 @@ Deno.test("validatePackageRequirement - number constraint", () => { const result = validatePackageRequirement("pkgx.sh/test", 1) assertEquals(result?.constraint.toString(), "^1") }) + +Deno.test("find", async () => { + useTestConfig() + const foo = await usePantry().find("python@3.11") + assertEquals(foo.length, 1) + assertEquals(foo[0].project, "python.org") +}) \ No newline at end of file diff --git a/src/hooks/usePantry.ts b/src/hooks/usePantry.ts index 8b96da2..183c1c5 100644 --- a/src/hooks/usePantry.ts +++ b/src/hooks/usePantry.ts @@ -1,10 +1,12 @@ import { is_what, PlainObject } from "../deps.ts" const { isNumber, isPlainObject, isString, isArray, isPrimitive, isBoolean } = is_what import { Package, Installation, PackageRequirement } from "../types.ts" +import { provides as cache_provides, available as cache_available, runtime_env as cache_runtime_env, companions as cache_companions, dependencies as cache_dependencies } from "./useSyncCache.ts"; import SemVer, * as semver from "../utils/semver.ts" import useMoustaches from "./useMoustaches.ts" import { PkgxError } from "../utils/error.ts" import { validate } from "../utils/misc.ts" +import * as pkgutils from "../utils/pkg.ts" import useConfig from "./useConfig.ts" import host from "../utils/host.ts" import Path from "../utils/Path.ts" @@ -45,6 +47,7 @@ export class PantryNotFoundError extends PantryError { export default function usePantry() { const prefix = useConfig().data.join("pantry/projects") + const is_cache_available = cache_available() && pantry_paths().length == 1 async function* ls(): AsyncGenerator { const seen = new Set() @@ -78,11 +81,23 @@ export default function usePantry() { throw new PackageNotFoundError(project) })() - const companions = async () => parse_pkgs_node((await yaml())["companions"]) + const companions = async () => { + if (is_cache_available) { + return await cache_companions(project) ?? parse_pkgs_node((await yaml())["companions"]) + } else { + return parse_pkgs_node((await yaml())["companions"]) + } + } const runtime_env = async (version: SemVer, deps: Installation[]) => { - const yml = await yaml() - const obj = validate.obj(yml["runtime"]?.["env"] ?? {}) + const obj = await (async () => { + if (is_cache_available) { + const cached = await cache_runtime_env(project) + if (cached) return cached + } + const yml = await yaml() + return validate.obj(yml["runtime"]?.["env"] ?? {}) + })() return expand_env_obj(obj, { project, version }, deps) } @@ -94,7 +109,13 @@ export default function usePantry() { return platforms.includes(host().platform) ||platforms.includes(`${host().platform}/${host().arch}`) } - const drydeps = async () => parse_pkgs_node((await yaml()).dependencies) + const drydeps = async () => { + if (is_cache_available) { + return await cache_dependencies(project) ?? parse_pkgs_node((await yaml()).dependencies) + } else { + return parse_pkgs_node((await yaml()).dependencies) + } + } const provides = async () => { let node = (await yaml())["provides"] @@ -164,6 +185,25 @@ export default function usePantry() { async function find(name: string) { type Foo = ReturnType & LsEntry + //lol FIXME + name = pkgutils.parse(name).project + + if (prefix.join(name).isDirectory()) { + const foo = project(name) + return [{...foo, project: name }] + } + + /// only use cache if PKGX_PANTRY_PATH is not set + if (is_cache_available) { + const cached = await cache_provides(name) + if (cached) { + return cached.map(x => ({ + ...project(x), + project: x + })) + } + } + name = name.toLowerCase() //TODO not very performant due to serial awaits @@ -234,7 +274,8 @@ export default function usePantry() { parse_pkgs_node, expand_env_obj, missing, - neglected + neglected, + pantry_paths } function pantry_paths(): Path[] { diff --git a/src/hooks/useSync.test.ts b/src/hooks/useSync.test.ts index 42195d9..ec99a1b 100644 --- a/src/hooks/useSync.test.ts +++ b/src/hooks/useSync.test.ts @@ -1,44 +1,53 @@ +import specimen, { _internals } from "./useSync.ts" import { useTestConfig } from "./useTestConfig.ts" +import * as mock from "deno/testing/mock.ts" import { assert } from "deno/assert/mod.ts" import usePantry from "./usePantry.ts" -import useSync from "./useSync.ts" // NOTE actually syncs from github // TODO unit tests should not do actual network calls, instead make an implementation suite Deno.test("useSync", async runner => { - await runner.step("w/o git", async () => { - const conf = useTestConfig({}) - usePantry().prefix.rm({ recursive: true }) // we need to delete the fixtured pantry - assert(conf.git === undefined) - await test() - }) - - await runner.step({ - name: "w/git", - ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"), - async fn() { - const conf = useTestConfig({ PATH: "/usr/bin" }) + const stub = mock.stub(_internals, "cache", async () => {}) + + try { + await runner.step("w/o git", async () => { + const conf = useTestConfig({}) usePantry().prefix.rm({ recursive: true }) // we need to delete the fixtured pantry - assert(conf.git !== undefined) + assert(conf.git === undefined) await test() + }) - // test the “already cloned” code-path - await useSync() - } - }) - - async function test() { - let errord = false - try { - await usePantry().project("gnu.org/gcc").available() - } catch { - errord = true - } - assert(errord, `should be no pantry but there is! ${usePantry().prefix}`) + await runner.step({ + name: "w/git", + ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"), + async fn() { + const conf = useTestConfig({ PATH: "/usr/bin" }) + usePantry().prefix.rm({ recursive: true }) // we need to delete the fixtured pantry + assert(conf.git !== undefined) + await test() + + // test the “already cloned” code-path + await specimen() + } + }) - await useSync() + async function test() { + let errord = false + try { + await usePantry().project("gnu.org/gcc").available() + } catch { + errord = true + } + assert(errord, `should be no pantry but there is! ${usePantry().prefix}`) - assert(await usePantry().project("gnu.org/gcc").available()) + await specimen() + + assert(await usePantry().project("gnu.org/gcc").available()) + } + + } finally { + stub.restore() } + }) diff --git a/src/hooks/useSync.ts b/src/hooks/useSync.ts index 55b4dfd..b1e4087 100644 --- a/src/hooks/useSync.ts +++ b/src/hooks/useSync.ts @@ -7,12 +7,14 @@ import useDownload from "./useDownload.ts" import usePantry from "./usePantry.ts" import useConfig from "./useConfig.ts" import Path from "../utils/Path.ts" +import useSyncCache from "./useSyncCache.ts"; //FIXME tar is fetched from PATH :/ we want control //FIXME run in general is not controllable since it delegates to the shell interface Logger { syncing(path: Path): void + caching(path: Path): void syncd(path: Path): void } @@ -23,6 +25,27 @@ export default async function(logger?: Logger) { const unflock = await flock(pantry_dir.mkdir('p')) + try { + await _internals.sync(pantry_dir) + try { + logger?.caching(pantry_dir) + await _internals.cache() + } catch (err) { + console.warn("failed to cache pantry") + console.error(err) + } + } finally { + await unflock() + } + + logger?.syncd(pantry_dir) +} + +export const _internals = { + sync, cache: useSyncCache +} + +async function sync(pantry_dir: Path) { try { //TODO if there was already a lock, just wait on it, don’t do the following stuff @@ -55,11 +78,7 @@ export default async function(logger?: Logger) { proc.close() - } finally { - await unflock() } - - logger?.syncd(pantry_dir) } //////////////////////// utils diff --git a/src/hooks/useSyncCache.node.ts b/src/hooks/useSyncCache.node.ts new file mode 100644 index 0000000..0c35178 --- /dev/null +++ b/src/hooks/useSyncCache.node.ts @@ -0,0 +1,31 @@ +// the sqlite lib we use only works in deno + +import { PackageRequirement } from "../../mod.ts"; + +export default async function() +{} + +export function provides(_program: string): string[] { + throw new Error() +} + +export function dependencies(_project: string): PackageRequirement[] { + throw new Error() +} + +export function completion(_prefix: string): string[] { + throw new Error() +} + +/// is the cache available? +export function available(): boolean { + return false +} + +export function companions(_project: string): PackageRequirement[] { + throw new Error() +} + +export function runtime_env(_project: string): Record { + throw new Error() +} diff --git a/src/hooks/useSyncCache.test.ts b/src/hooks/useSyncCache.test.ts new file mode 100644 index 0000000..1a8f9fa --- /dev/null +++ b/src/hooks/useSyncCache.test.ts @@ -0,0 +1,26 @@ +import specimen, { provides, dependencies, available, runtime_env, completion, companions } from "./useSyncCache.ts" +import { useTestConfig } from "./useTestConfig.ts" +import { assert, assertEquals } from "deno/assert/mod.ts" +import { _internals } from "./useSync.ts" +import usePantry from "./usePantry.ts" + +// NOTE actually syncs from github +// TODO unit tests should not do actual network calls, instead make an implementation suite + +Deno.test({ + name: "useSyncCache", + ignore: Deno.build.os == 'windows', + async fn() { + useTestConfig() + await _internals.sync(usePantry().prefix.parent()) + await specimen() + + //TODO test better + assert(available()) + assertEquals((await provides('node'))?.[0], 'nodejs.org') + // assertEquals((await dependencies('nodejs.org'))?.length, 3) + assert(new Set(await completion('nod')).has("node")) + assertEquals((await companions("nodejs.org"))?.[0]?.project, "npmjs.com") + assert((await runtime_env("numpy.org"))?.["PYTHONPATH"]) + } +}) diff --git a/src/hooks/useSyncCache.ts b/src/hooks/useSyncCache.ts new file mode 100644 index 0000000..1665387 --- /dev/null +++ b/src/hooks/useSyncCache.ts @@ -0,0 +1,181 @@ +import { Database } from "https://deno.land/x/sqlite3@0.10.0/mod.ts"; +import * as pkgutils from "../utils/pkg.ts"; +import usePantry from "./usePantry.ts"; +import useConfig from "./useConfig.ts"; + +export default async function() { + if (Deno.build.os == 'windows') return + + const path = useConfig().cache.join('pantry.db').rm() // delete it first so pantry instantiation doesn't use cache + const { ls, ...pantry } = usePantry() + + const sqlite = await install_sqlite() + if (!sqlite) return + Deno.env.set("DENO_SQLITE_PATH", sqlite.string) + const db = new Database(path.string) + + // unique or don’t insert what is already there or just dump tables first perhaps + + try { + await db.transaction(async () => { + db.exec(` + DROP TABLE IF EXISTS provides; + DROP TABLE IF EXISTS dependencies; + DROP TABLE IF EXISTS companions; + DROP TABLE IF EXISTS runtime_env; + CREATE TABLE provides ( + project TEXT, + program TEXT + ); + CREATE TABLE dependencies ( + project TEXT, + pkgspec TEXT + ); + CREATE TABLE companions ( + project TEXT, + pkgspec TEXT + ); + CREATE TABLE runtime_env ( + project TEXT, + envline TEXT + ); + CREATE INDEX idx_project ON provides(project); + CREATE INDEX idx_program ON provides(program); + CREATE INDEX idx_project_dependencies ON dependencies(project); + CREATE INDEX idx_project_companions ON companions(project); + `); + + for await (const pkg of ls()) { + if (!pkg.path.string.startsWith(pantry.prefix.string)) { + // don’t cache PKGX_PANTRY_PATH additions + continue; + } + + try { + const project = pantry.project(pkg.project) + const [programs, deps, companions, yaml] = await Promise.all([ + project.provides(), + project.runtime.deps(), + project.companions(), + project.yaml() + ]) + + for (const program of programs) { + db.exec(`INSERT INTO provides (project, program) VALUES ('${pkg.project}', '${program}');`); + } + + for (const dep of deps) { + db.exec(`INSERT INTO dependencies (project, pkgspec) VALUES ('${pkg.project}', '${pkgutils.str(dep)}')`); + } + + for (const companion of companions) { + db.exec(`INSERT INTO companions (project, pkgspec) VALUES ('${pkg.project}', '${pkgutils.str(companion)}')`); + } + + for (const [key, value] of Object.entries(yaml.runtime?.env ?? {})) { + db.exec(`INSERT INTO runtime_env (project, envline) VALUES ('${pkg.project}', '${key}=${value}')`); + } + } catch { + console.warn("corrupt yaml", pkg.path) + } + } + })() + } catch (err) { + path.rm() + throw err + } finally { + db.close(); + } +} + +export async function provides(program: string) { + const db = await _db() + if (!db) return + try { + return db.sql`SELECT project FROM provides WHERE program = ${program}`.map(x => x.project); + } finally { + db.close() + } +} + +export async function dependencies(project: string) { + const db = await _db() + if (!db) return + try { + return db.sql`SELECT pkgspec FROM dependencies WHERE project = ${project}`.map(x => pkgutils.parse(x.pkgspec)); + } finally { + db.close() + } +} + +export async function completion(prefix: string) { + const db = await _db() + try { + return db?.prepare(`SELECT program FROM provides WHERE program LIKE '${prefix}%'`).value<[string]>()!; + } finally { + db?.close() + } +} + +/// is the cache available? +export function available() { + if (Deno.build.os == 'windows') { + return false + } else { + const path = useConfig().cache.join('pantry.db') + return path.isFile() + } +} + +export async function companions(project: string) { + const db = await _db() + if (!db) return + try { + return db.sql`SELECT pkgspec FROM companions WHERE project = ${project}`.map(x => pkgutils.parse(x.pkgspec)); + } finally { + db.close() + } +} + +export async function runtime_env(project: string) { + const db = await _db() + if (!db) return + try { + const rv: Record = {} + for (const {envline: line} of db.sql`SELECT envline FROM runtime_env WHERE project = ${project}`) { + const [key, ...rest] = line.split("=") + rv[key] = rest.join('=') + } + return rv + } finally { + db.close() + } +} + +import useCellar from "./useCellar.ts" + +async function _db() { + if (Deno.build.os == 'windows') return + const path = useConfig().cache.join('pantry.db') + if (!path.isFile()) return + const sqlite = await useCellar().has({ project: "sqlite.org", constraint: new semver.Range('*') }) + if (!sqlite) return + const ext = host().platform == 'darwin' ? 'dylib' : 'so' + Deno.env.set("DENO_SQLITE_PATH", sqlite.path.join(`lib/libsqlite3.${ext}`).string) + return new Database(path.string) +} + +import install from "../porcelain/install.ts" +import host from "../utils/host.ts"; +import Path from "../utils/Path.ts"; +import { semver } from "../../mod.ts"; + +async function install_sqlite(): Promise { + const foo = await install("sqlite.org") + for (const bar of foo) { + if (bar.pkg.project == 'sqlite.org') { + const ext = host().platform == 'darwin' ? 'dylib' : 'so' + return bar.path.join(`lib/libsqlite3.${ext}`) + } + } +} \ No newline at end of file diff --git a/src/hooks/useTestConfig.ts b/src/hooks/useTestConfig.ts index 87baa8e..b1bd41c 100644 --- a/src/hooks/useTestConfig.ts +++ b/src/hooks/useTestConfig.ts @@ -1,5 +1,5 @@ import useConfig, { ConfigDefault } from "./useConfig.ts" -import { fromFileUrl } from "deno/path/from_file_url.ts" +import { fromFileUrl } from "deno/path/mod.ts" import Path from "../utils/Path.ts" export function useBaseTestConfig(env?: Record) { diff --git a/src/plumbing/which.ts b/src/plumbing/which.ts index c69d47d..a63568a 100644 --- a/src/plumbing/which.ts +++ b/src/plumbing/which.ts @@ -1,12 +1,12 @@ -import { PackageRequirement } from "../types.ts" +import { provides as cache_provides, available as cache_available } from "../hooks/useSyncCache.ts" import usePantry, { PantryError } from "../hooks/usePantry.ts" +import { PackageRequirement } from "../types.ts" import * as semver from "../utils/semver.ts" export type WhichResult = PackageRequirement & { shebang: string[] } - export default async function which(arg0: string, opts?: { providers?: boolean }): Promise; export default async function which(arg0: string, opts: { providers?: boolean, all: false }): Promise; export default async function which(arg0: string, opts: { providers?: boolean, all: true }): Promise; @@ -36,6 +36,19 @@ async function *_which(arg0: string, opts: { providers: boolean }): AsyncGenerat const pantry = usePantry() let found: WhichResult[] = [] + + // don't use the cache if PKGX_PANTRY_PATH is set + if (cache_available()) { + const cached = await cache_provides(arg0) + if (cached) { + for (const project of cached) { + yield { project, constraint: new semver.Range("*"), shebang: [arg0] } + } + // NOTE probs wrong, but we need a rewrite + if (cached.length) return + } + } + const promises: Promise[] = [] for await (const entry of pantry.ls()) { @@ -92,9 +105,10 @@ async function *_which(arg0: string, opts: { providers: boolean }): AsyncGenerat } } + await Promise.all(promises) + // if we didn’t find anything yet then we have to wait on the promises // otherwise we can ignore them - await Promise.all(promises) for (const f of found) { yield f