.some

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

.github/scripts/api.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env -S pkgx deno run --allow-read
+
+import { basename } from "https://deno.land/[email protected]/path/basename.ts";
+import * as flags from "https://deno.land/[email protected]/flags/mod.ts";
+
+interface Script {
+  fullname: string
+  birthtime: Date
+  description?: string
+  avatar: string
+  url: string
+}
+
+interface ApiScript {
+  name: string
+  fullname: string
+  birthtime: Date
+  description?: string
+  avatar: string
+  url: string
+}
+
+const args = flags.parse(Deno.args);
+const input_json = args['current-api-json']
+const index_json = args['index-json']
+
+if (!input_json) throw new Error("--current-api-json must be set to the existing api.json file")
+if (!index_json) throw new Error("--index-json must be set to the existing index.json file")
+
+const existing = (JSON.parse(Deno.readTextFileSync(input_json)).scripts as ApiScript[]).reduce((acc, cur) => {
+  acc[cur.name || cur.fullname] = cur
+  return acc
+}, {} as Record<string, ApiScript>)
+const index = JSON.parse(Deno.readTextFileSync(index_json)).scripts as ApiScript[]
+const rv: ApiScript[] = []
+
+for (const script of index as Script[]) {
+  const base = basename(script.fullname)
+  if (existing[base] && existing[base].fullname == script.fullname) {
+    rv.push(convert(script, base))
+  } else if (existing[script.fullname]) {
+    rv.push(convert(script, script.fullname))
+  } else if (!existing[base]) {
+    /// new owner of the basename for this script
+    rv.push(convert(script, base))
+  } else {
+    /// someone already claimed this basename
+    rv.push(convert(script, script.fullname))
+  }
+}
+
+console.log(JSON.stringify({scripts: rv}, null, 2))
+
+
+function convert(script: Script, name: string): ApiScript {
+  const url = script.url.replace('github.com', 'raw.githubusercontent.com').replace('/blob', '');
+
+  return {
+    name,
+    fullname: script.fullname,
+    birthtime: script.birthtime,
+    description: script.description,
+    avatar: script.avatar,
+    url,
+  }
+}

.github/scripts/index.ts

@@ -0,0 +1,128 @@
+#!/usr/bin/env -S pkgx deno run --allow-run --allow-read=.
+
+import { join, basename, dirname } from "https://deno.land/[email protected]/path/mod.ts";
+import { walk, exists } from "https://deno.land/[email protected]/fs/mod.ts";
+import * as flags from "https://deno.land/[email protected]/flags/mod.ts";
+
+const args = flags.parse(Deno.args);
+const outdir = args['out']
+
+if (!outdir) {
+  console.error(`usage: index.ts --out <path>`);
+  Deno.exit(1);
+}
+
+Deno.chdir(outdir);
+
+const rv: Script[] = []
+for await (const gitRepoPath of iterateGitRepos('.')) {
+  console.error(`iterating: ${gitRepoPath}`);
+  rv.push(...await get_metadata(gitRepoPath));
+}
+
+rv.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime());
+
+console.log(JSON.stringify({ scripts: rv }, null, 2));
+
+
+////////////////////////////////////////////////////////////////////// lib
+async function extractMarkdownSection(filePath: string, sectionTitle: string): Promise<string | undefined> {
+  const data = await Deno.readTextFile(filePath);
+  const lines = data.split('\n');
+  let capturing = false;
+  let sectionContent = '';
+
+  for (const line of lines) {
+    if (line.startsWith('## ')) {
+      if (capturing) break;  // stop if we reach another ## section
+      if (normalize_title(line.slice(3)) == normalize_title(sectionTitle)) capturing = true;
+    } else if (capturing) {
+      sectionContent += line + '\n';
+    }
+  }
+
+  return chuzzle(sectionContent);
+
+  function normalize_title(input: string) {
+    return input.toLowerCase().replace(/[^a-z0-9]/g, '').trim();
+  }
+}
+
+interface Script {
+  fullname: string
+  birthtime: Date
+  description?: string
+  avatar: string
+  url: string
+}
+
+async function* iterateGitRepos(basePath: string): AsyncIterableIterator<string> {
+  for await (const entry of walk(basePath, { maxDepth: 2 })) {
+    if (entry.isDirectory && await exists(join(entry.path, '.git'))) {
+      yield entry.path;
+    }
+  }
+}
+
+function chuzzle(ln: string): string | undefined {
+  const out = ln.trim()
+  return out || undefined;
+}
+
+async function get_metadata(slug: string) {
+
+  const cmdString = `git -C '${slug}' log --pretty=format:'%H %aI' --name-only --diff-filter=A -- scripts`;
+
+  const process = Deno.run({
+    cmd: ["bash", "-c", cmdString],
+    stdout: "piped"
+  });
+
+  const output = new TextDecoder().decode(await process.output());
+  await process.status();
+  process.close();
+
+  const lines = chuzzle(output)?.split('\n') ?? [];
+  const rv: Script[] = []
+  let currentCommitDate: string | undefined;
+
+  for (let line of lines) {
+    line = line.trim()
+
+    if (line.includes(' ')) {  // Detect lines with commit hash and date
+      currentCommitDate = line.split(' ')[1];
+    } else if (line && currentCommitDate) {
+      const filename = join(slug, line)
+      if (!await exists(filename)) {
+        // the file used to exist but has been deleted
+        console.warn("skipping deleted: ", filename)
+        continue
+      }
+
+      console.error(line)
+
+      const repo_metadata = JSON.parse(await Deno.readTextFile(join(slug, 'metadata.json')))
+
+      const description = await extractMarkdownSection(join(slug, 'README.md'), basename(filename));
+      const birthtime = new Date(currentCommitDate!);
+      const avatar = repo_metadata.avatar
+      const fullname = join(dirname(slug), ...stem(filename))
+      // const excerpt = (await Deno.readTextFile(filename)).split("\n").slice(0, 5).join("\n")
+      const url = repo_metadata.url +'/scripts/' + basename(filename)
+
+      rv.push({ fullname, birthtime, description, avatar, url })
+    }
+  }
+
+  return rv;
+
+  function stem(filename: string): string[] {
+    const base = basename(filename)
+    const parts = base.split('.')
+    if (parts.length == 1) {
+      return parts.slice(0, 1)
+    } else {
+      return parts.slice(0, -1) // no extension, but allow eg. foo.bar.js to be foo.bar
+    }
+  }
+}

.github/scripts/redirects.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env -S pkgx deno run --allow-write=. --allow-read=.
+
+import * as flags from "https://deno.land/[email protected]/flags/mod.ts";
+
+interface ApiScript {
+  name: string
+  fullname: string
+  birthtime: Date
+  description?: string
+  avatar: string
+  url: string
+}
+
+const args = flags.parse(Deno.args)
+const outdir = args['out']
+const api_json = args['api-json']
+const scripts = JSON.parse(await Deno.readTextFile(api_json)).scripts as ApiScript[]
+
+for (const {name, fullname} of scripts) {
+  if (name == fullname) continue
+  Deno.copyFileSync(`${outdir}/${fullname}`, `${outdir}/${name}`)
+}

.github/scripts/trawl.ts

@@ -0,0 +1,55 @@
+#!/usr/bin/env -S pkgx deno run --allow-run --allow-net --allow-env=GH_TOKEN --allow-write=.
+
+import * as flags from "https://deno.land/[email protected]/flags/mod.ts";
+
+const args = flags.parse(Deno.args);
+const outdir = args['out']
+
+const ghToken = Deno.env.get("GH_TOKEN");
+if (!ghToken) {
+  console.error("error: GitHub token is required. Set the GH_TOKEN environment variable.");
+  Deno.exit(1)
+}
+
+Deno.mkdirSync(outdir, { recursive: true });
+
+async function cloneAllForks(user: string, repo: string) {
+  let page = 1;
+  while (true) {
+    const response = await fetch(`https://api.github.com/repos/${user}/${repo}/forks?page=${page}`, {
+      headers: {
+        "Authorization": `token ${ghToken}`
+      }
+    });
+
+    if (!response.ok) {
+      throw new Error(`err: ${response.statusText}`);
+    }
+
+    const forks = await response.json();
+    if (forks.length === 0) {
+      break;  // No more forks
+    }
+
+    for (const fork of forks) {
+      const cloneUrl = fork.clone_url;
+      console.log(`Cloning ${cloneUrl}...`);
+      const proc = new Deno.Command("git", { args: ["-C", outdir, "clone", cloneUrl, `${fork.full_name}`]}).spawn()
+      if (!(await proc.status).success) {
+        throw new Error(`err: ${await proc.status}`)
+      }
+
+      Deno.writeTextFileSync(`${outdir}/${fork.full_name}/metadata.json`, JSON.stringify({
+        stars: fork.stargazers_count,
+        license: fork.license.spdx_id,
+        avatar: fork.owner.avatar_url,
+        url: fork.html_url + '/blob/' + fork.default_branch
+      }, null, 2))
+    }
+
+    page++;
+  }
+}
+
+// Example usage:
+await cloneAllForks('pkgxdev', 'scripthub');

.github/workflows/deploy.yml

@@ -0,0 +1,71 @@
+on:
+  push:
+    branches: main
+    paths:
+      - .github/workflows/deploy.yml
+      - .github/scripts/*
+  pull_request:
+    paths:
+      - .github/workflows/deploy.yml
+  schedule:
+    - cron: '23 * * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    if: github.repository == 'pkgxdev/scripthub'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pkgxdev/setup@v2
+
+      - run: .github/scripts/trawl.ts --out ./out
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - run: .github/scripts/index.ts --out ./out > ./out/index.json
+
+      # safe to handle multiple forks but this is not supported and the metadata will not be merged
+      - run: |
+          for x in *; do
+            if [ -d $x ]; then
+              cd $x
+              mv */scripts/* .
+              mv */metadata.json .
+              find -type d -depth 1 | xargs rm -rf
+              cd ..
+            fi
+          done
+        working-directory: out
+
+      # FIXME there is a possible race condition here since github pages
+      # deploys slowly. There’s no fix without switching to using a branch for gh-pages
+      # or another deployment solution altogether. NOTE we want to do one or both in the near future.
+      - run: |
+          curl -LfO https://pkgxdev.github.io/scripthub/api.json
+          .github/scripts/api.ts --current-api-json ./api.json --index-json ./out/index.json > out/api.json
+
+      - run: .github/scripts/redirects.ts --out ./out --api-json ./out/api.json
+
+      - uses: actions/configure-pages@v3
+      - uses: actions/upload-pages-artifact@v2
+        with:
+          path: out
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    permissions:
+      pages: write      # to deploy to Pages
+      id-token: write   # to verify the deployment originates from an appropriate source
+    steps:
+      - uses: actions/deploy-pages@v2
+        id: deployment

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+/out

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "deno.enable": true,
+  "deno.lint": true,
+  "deno.unstable": true
+}