refactor!: new data structure

- codes are now detatched from actual app runtime - scraper cli as npx command
rayriffy · Feb 10, 2024 · 84f23f1 · 84f23f1
1 parent 1290b31
commit 84f23f1
Show file tree

Hide file tree

Showing 192 changed files with 459 additions and 4,607 deletions.
diff --git a/.github/workflows/deploys.yml b/.github/workflows/deploys.yml
diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml
@@ -0,0 +1,27 @@
+name: runtime
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: login
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: rayriffy
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: buildx
+        uses: docker/setup-buildx-action@v2
+      - name: build-and-push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/rayriffy/riffyh-runtime
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,8 @@
 .DS_Store
 node_modules
-/build
-/.svelte-kit
-/package
+build
+.svelte-kit
+package
 .env
 .env.*
 !.env.example
@@ -14,4 +14,5 @@ data
 .urami
 stats.html
 pnpm-lock.yaml
-yarn.lock
+yarn.lock
+dist
diff --git a/bun.lockb b/bun.lockb
diff --git a/commons/package.json b/commons/package.json
@@ -0,0 +1,5 @@
+{
+  "name": "@riffyh/commons",
+  "main": "src/index.ts",
+  "private": true
+}
diff --git a/src/core/@types/DatabaseCode.ts → commons/src/@types/DatabaseCode.ts b/src/core/@types/DatabaseCode.ts → commons/src/@types/DatabaseCode.ts
diff --git a/src/core/@types/DatabaseTag.ts → commons/src/@types/DatabaseTag.ts b/src/core/@types/DatabaseTag.ts → commons/src/@types/DatabaseTag.ts
diff --git a/src/core/@types/Hentai.ts → commons/src/@types/Hentai.ts b/src/core/@types/Hentai.ts → commons/src/@types/Hentai.ts
diff --git a/src/core/@types/HifuminHentai.ts → commons/src/@types/HifuminHentai.ts b/src/core/@types/HifuminHentai.ts → commons/src/@types/HifuminHentai.ts
diff --git a/src/core/@types/HifuminImage.ts → commons/src/@types/HifuminImage.ts b/src/core/@types/HifuminImage.ts → commons/src/@types/HifuminImage.ts
diff --git a/src/core/@types/HifuminSingleResponse.ts → commons/src/@types/HifuminSingleResponse.ts b/src/core/@types/HifuminSingleResponse.ts → commons/src/@types/HifuminSingleResponse.ts
diff --git a/src/core/@types/HifuminTag.ts → commons/src/@types/HifuminTag.ts b/src/core/@types/HifuminTag.ts → commons/src/@types/HifuminTag.ts
diff --git a/src/core/@types/Image.ts → commons/src/@types/Image.ts b/src/core/@types/Image.ts → commons/src/@types/Image.ts
diff --git a/src/core/@types/NHHentai.ts → commons/src/@types/NHHentai.ts b/src/core/@types/NHHentai.ts → commons/src/@types/NHHentai.ts
diff --git a/src/core/@types/Tag.ts → commons/src/@types/Tag.ts b/src/core/@types/Tag.ts → commons/src/@types/Tag.ts
diff --git a/src/core/@types/TagType.ts → commons/src/@types/TagType.ts b/src/core/@types/TagType.ts → commons/src/@types/TagType.ts
diff --git a/src/core/constants/hifuminHentaiQuery.ts → commons/src/constants/hifuminHentaiQuery.ts b/src/core/constants/hifuminHentaiQuery.ts → commons/src/constants/hifuminHentaiQuery.ts
diff --git a/src/core/constants/itemsPerPage.ts → commons/src/constants/itemsPerPage.ts b/src/core/constants/itemsPerPage.ts → commons/src/constants/itemsPerPage.ts
diff --git a/tools/backup.ts → commons/src/functions/getHentaiFromNH.ts b/tools/backup.ts → commons/src/functions/getHentaiFromNH.ts
@@ -1,15 +1,15 @@
-import kebabCase from 'lodash.kebabcase'
+import kebabCase from 'lodash/kebabCase'
 
-import type { Hentai } from '../src/core/@types/Hentai'
+import type { Hentai } from '../@types/Hentai'
 
-export const backupFetch = async (id: string | number): Promise<Hentai> => {
+export const getHentaiFromNH = async (id: string | number): Promise<Hentai> => {
   const data = await fetch(`https://nhentai.net/api/gallery/${id}`).then(o => {
     if (o.ok) return o.json() as Promise<Hentai>
     else throw o
   })
 
   return {
-    id: data.id,
+    id: Number(data.id),
     media_id: data.media_id,
     title: data.title,
     images: {

diff --git a/src/core/services/hifuminHentaiToHentai.ts → ...ns/src/functions/hifuminHentaiToHentai.ts b/src/core/services/hifuminHentaiToHentai.ts → ...ns/src/functions/hifuminHentaiToHentai.ts
@@ -1,4 +1,4 @@
-import kebabCase from 'lodash.kebabcase'
+import kebabCase from 'lodash/kebabCase'
 
 import type { Hentai } from '../@types/Hentai'
 import type { Image } from '../@types/Image'

diff --git a/commons/src/index.ts b/commons/src/index.ts
@@ -0,0 +1,20 @@
+// constants
+export { hifuminHentaiQuery } from './constants/hifuminHentaiQuery'
+export { itemsPerPage } from './constants/itemsPerPage'
+
+// functions
+export { getHentaiFromNH } from './functions/getHentaiFromNH'
+export { hifuminHentaiToHentai } from './functions/hifuminHentaiToHentai'
+
+// types
+export type { DatabaseCode } from './@types/DatabaseCode'
+export type { DatabaseTag } from './@types/DatabaseTag'
+export type { Hentai } from './@types/Hentai'
+export type { HifuminHentai } from './@types/HifuminHentai'
+export type { HifuminImage } from './@types/HifuminImage'
+export type { HifuminSingleResponse } from './@types/HifuminSingleResponse'
+export type { HifuminTag } from './@types/HifuminTag'
+export type { Image } from './@types/Image'
+export type { NHHentai } from './@types/NHHentai'
+export type { Tag } from './@types/Tag'
+export type { TagType } from './@types/TagType'
diff --git a/package.json b/package.json
@@ -1,83 +1,19 @@
 {
-  "name": "svelte",
-  "version": "0.0.1",
-  "author": "Phumrapee Limpianchop <[email protected]>",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/rayriffy/rayriffy-h.git"
-  },
-  "bugs": {
-    "url": "https://github.com/rayriffy/rayriffy-h/issues"
-  },
-  "homepage": "https://h.rayriffy.com",
-  "license": "MIT",
-  "private": true,
-  "scripts": {
-    "prepare": "prisma generate || true",
-    "dev": "vite dev",
-    "build": "vite build && bun tools/patchSW.ts",
-    "preview": "vite preview",
-    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
-    "format": "prettier --write \"**/*.{ts,tsx,md,astro,svelte}\"",
-    "fetch:data": "bun tools/fetchData.ts",
-    "sync:data": "bun tools/syncData.ts",
-    "patch:sharp": "patch node_modules/sharp/install/libvips.js < patches/sharp.patch",
-    "prisma:generate": "prisma generate",
-    "prisma:commit": "prisma migrate dev",
-    "prisma:deploy": "prisma migrate deploy",
-    "sync": "bun run fetch:data && bun run sync:data"
-  },
-  "trustedDependencies": [
-    "prettier"
+  "name": "riffyh",
+  "version": "1.0.0",
+  "workspaces": [
+    "web/",
+    "scraper/",
+    "commons/"
   ],
-  "type": "module",
   "dependencies": {
-    "@nanostores/persistent": "0.9.1",
-    "@tanstack/svelte-query": "5.17.1",
-    "@trpc/client": "10.45.0",
-    "@trpc/server": "10.45.0",
-    "@urami/core": "1.2.3",
-    "@urami/svelte": "1.1.1",
-    "cookie": "0.6.0",
-    "destr": "2.0.2",
-    "devalue": "4.3.2",
-    "lodash.kebabcase": "4.1.1",
-    "nanostores": "0.9.5",
-    "p-queue": "7.4.1",
-    "set-cookie-parser": "2.6.0",
-    "sharp": "0.33.1",
-    "trpc-svelte-query": "2.0.5",
-    "zod": "3.22.4"
+    "lodash": "4.17.21"
   },
   "devDependencies": {
-    "@prisma/client": "5.7.1",
-    "@sveltejs/kit": "2.0.6",
-    "@sveltejs/vite-plugin-svelte": "3.0.1",
-    "@types/cookie": "0.6.0",
-    "@types/lodash.kebabcase": "4.1.9",
+    "@types/lodash": "4.14.202",
     "@types/node": "20.10.6",
-    "@vite-pwa/sveltekit": "0.3.0",
-    "autoprefixer": "10.4.16",
     "bun-types": "1.0.21",
-    "daisyui": "4.5.0",
-    "postcss": "8.4.32",
-    "prettier": "3.1.1",
-    "prettier-plugin-svelte": "3.1.2",
-    "prettier-plugin-tailwindcss": "0.5.10",
-    "prisma": "5.7.1",
-    "rollup-plugin-visualizer": "5.12.0",
-    "svelte": "4.2.8",
-    "svelte-adapter-bun": "0.5.1",
-    "svelte-check": "3.6.2",
-    "tailwindcss": "3.4.0",
     "tslib": "2.6.2",
-    "typescript": "5.3.3",
-    "vite": "5.0.10",
-    "vite-plugin-pwa": "0.17.4",
-    "workbox-window": "7.0.0"
-  },
-  "overrides": {
-    "@rollup/plugin-node-resolve": "15.2.3",
-    "sharp": "0.33.1"
+    "typescript": "5.3.3"
   }
-}
+}
diff --git a/prisma/migrations/20220830171401_initial_c/migration.sql b/prisma/migrations/20220830171401_initial_c/migration.sql
diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
diff --git a/scraper/package.json b/scraper/package.json
@@ -0,0 +1,30 @@
+{
+  "name": "@riffyh/scraper",
+  "version": "1.0.2",
+  "bin": "dist/index.js",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rayriffy/rayriffy-h.git"
+  },
+  "scripts": {
+    "build": "bun build src/index.ts --outfile dist/index.js --target bun -e kysely -e pg -e p-queue -e yargs"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "kysely": "0.27.2",
+    "pg": "8.11.3",
+    "p-queue": "8.0.1",
+    "yargs": "17.7.2"
+  },
+  "devDependencies": {
+    "@riffyh/commons": "workspace:*",
+    "@types/pg": "8.11.0",
+    "@types/yargs": "17.0.32"
+  }
+}
diff --git a/scraper/src/@types/SQLDatabase.ts b/scraper/src/@types/SQLDatabase.ts
@@ -0,0 +1,8 @@
+import { Hentai } from "@riffyh/commons"
+
+export interface SQLDatabase {
+  Hentai: {
+    id: number
+    data: string
+  }
+}