From 505e5ee6bccc2615f421b9008f28c729367be151 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Sat, 23 Nov 2024 16:08:59 +0800 Subject: [PATCH] refactor: split packages --- docs/.vitepress/theme/index.ts | 4 +- docs/vite.config.ts | 2 +- .../src/main.ts | 4 +- examples/neuri/weather-query/src/valibot.ts | 6 +- examples/neuri/weather-query/src/zod.ts | 6 +- package.json | 1 + packages/format-code/package.json | 59 +++++++++++++++++++ .../src/textmate/index.test.ts} | 2 +- .../src/textmate/index.ts} | 3 +- packages/neuri/package.json | 14 ----- .../neuri/src/formats/json/jsonParser.test.ts | 2 +- packages/neuri/src/formats/json/jsonParser.ts | 7 ++- .../neuri/src/formats/json/jsonschema.test.ts | 2 +- .../src/formats/json/streamJsonParser.ts | 2 +- .../neuri/src/formats/markdown/markdown.ts | 6 +- packages/use-fs/package.json | 55 +++++++++++++++++ .../agents/fs => use-fs/src}/index.test.ts | 8 +-- .../src/agents/fs => use-fs/src}/index.ts | 10 ++-- .../src/agents/fs => use-fs/src}/utils.ts | 0 pnpm-lock.yaml | 54 ++++++++++++----- 20 files changed, 188 insertions(+), 59 deletions(-) create mode 100644 packages/format-code/package.json rename packages/{neuri/src/formats/code/textmate.test.ts => format-code/src/textmate/index.test.ts} (90%) rename packages/{neuri/src/formats/code/textmate.ts => format-code/src/textmate/index.ts} (99%) create mode 100644 packages/use-fs/package.json rename packages/{neuri/src/agents/fs => use-fs/src}/index.test.ts (100%) rename packages/{neuri/src/agents/fs => use-fs/src}/index.ts (98%) rename packages/{neuri/src/agents/fs => use-fs/src}/utils.ts (100%) diff --git a/docs/.vitepress/theme/index.ts b/docs/.vitepress/theme/index.ts index 1a3ced3..aaaed12 100644 --- a/docs/.vitepress/theme/index.ts +++ b/docs/.vitepress/theme/index.ts @@ -1,7 +1,7 @@ -import { h } from 'vue' - import type { Theme } from 'vitepress' + import DefaultTheme from 'vitepress/theme' +import { h } from 'vue' import 'virtual:uno.css' diff --git a/docs/vite.config.ts b/docs/vite.config.ts index 603c245..9c34776 100644 --- a/docs/vite.config.ts +++ b/docs/vite.config.ts @@ -1,5 +1,5 @@ -import { defineConfig } from 'vite' import UnoCSS from 'unocss/vite' +import { defineConfig } from 'vite' import Inspect from 'vite-plugin-inspect' export default defineConfig({ diff --git a/examples/neuri/documentations-from-unit-tests/src/main.ts b/examples/neuri/documentations-from-unit-tests/src/main.ts index 07f1153..d5949ed 100644 --- a/examples/neuri/documentations-from-unit-tests/src/main.ts +++ b/examples/neuri/documentations-from-unit-tests/src/main.ts @@ -1,10 +1,10 @@ import { env } from 'node:process' -import OpenAI from 'openai' - import { composeAgent, } from 'neuri/openai' +import OpenAI from 'openai' + async function main() { const o = new OpenAI({ baseURL: env.OPENAI_API_BASEURL, diff --git a/examples/neuri/weather-query/src/valibot.ts b/examples/neuri/weather-query/src/valibot.ts index 6972a09..35eb827 100644 --- a/examples/neuri/weather-query/src/valibot.ts +++ b/examples/neuri/weather-query/src/valibot.ts @@ -1,7 +1,4 @@ import { env } from 'node:process' -import OpenAI from 'openai' -import * as v from 'valibot' - import { composeAgent, defineToolFunction, @@ -11,6 +8,9 @@ import { toolFunction, user, } from 'neuri/openai' +import OpenAI from 'openai' + +import * as v from 'valibot' async function main() { const o = new OpenAI({ diff --git a/examples/neuri/weather-query/src/zod.ts b/examples/neuri/weather-query/src/zod.ts index 9ff01d3..fa2ca82 100644 --- a/examples/neuri/weather-query/src/zod.ts +++ b/examples/neuri/weather-query/src/zod.ts @@ -1,7 +1,4 @@ import { env } from 'node:process' -import OpenAI from 'openai' -import * as z from 'zod' - import { composeAgent, defineToolFunction, @@ -11,6 +8,9 @@ import { toolFunction, user, } from 'neuri/openai' +import OpenAI from 'openai' + +import * as z from 'zod' async function main() { const o = new OpenAI({ diff --git a/package.json b/package.json index 0e8831d..2646514 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "scripts": { "stub": "pnpm -r --filter=./packages/* --parallel run stub", "lint": "eslint --cache .", + "lint:fix": "eslint --cache --fix .", "docs:dev": "pnpm -r --filter=./docs run dev", "docs:build": "pnpm -r --filter=./docs run build", "docs:preview": "pnpm -r --filter=./docs run preview", diff --git a/packages/format-code/package.json b/packages/format-code/package.json new file mode 100644 index 0000000..e2cc25d --- /dev/null +++ b/packages/format-code/package.json @@ -0,0 +1,59 @@ +{ + "name": "@neuri/format-code", + "type": "module", + "version": "0.0.5", + "description": "Code related output parsers for neuri", + "author": { + "name": "Lingtic.io Neuri Team", + "email": "neuridev@lingtic.io", + "url": "https://github.com/lingticio" + }, + "license": "MIT", + "homepage": "https://docs.lingtic.io/pages/en/neuri", + "repository": { + "type": "git", + "url": "https://github.com/lingticio/neuri-js.git", + "directory": "packages/format-code" + }, + "keywords": [ + "guiiai", + "llm", + "openai", + "lingtic", + "neuri" + ], + "exports": { + ".": { + "types": "./dist/textmate/index.d.ts", + "import": "./dist/textmate/index.mjs" + }, + "./textmate": { + "types": "./dist/textmate/index.d.ts", + "import": "./dist/textmate/index.mjs" + } + }, + "module": "./dist/textmate/index.mjs", + "types": "./dist/textmate/index.d.ts", + "files": [ + "LICENSE", + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "unbuild --stub", + "stub": "unbuild --stub", + "build": "unbuild", + "package:publish": "pnpm build && pnpm publish --access public --no-git-checks" + }, + "dependencies": { + "@guiiai/logg": "^1.0.3", + "@shikijs/core": "^1.14.1", + "@shikijs/vscode-textmate": "^9.2.2", + "shiki": "^1.14.1", + "tm-grammars": "^1.17.18" + }, + "devDependencies": { + "@types/node": "^20.11.17" + } +} diff --git a/packages/neuri/src/formats/code/textmate.test.ts b/packages/format-code/src/textmate/index.test.ts similarity index 90% rename from packages/neuri/src/formats/code/textmate.test.ts rename to packages/format-code/src/textmate/index.test.ts index 692f47e..d7df8ec 100644 --- a/packages/neuri/src/formats/code/textmate.test.ts +++ b/packages/format-code/src/textmate/index.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { tokenizeByTextMateGrammar } from './textmate' +import { tokenizeByTextMateGrammar } from '.' describe('tokenizeByTextMateGrammar', () => { it('should tokenize vue code', async () => { diff --git a/packages/neuri/src/formats/code/textmate.ts b/packages/format-code/src/textmate/index.ts similarity index 99% rename from packages/neuri/src/formats/code/textmate.ts rename to packages/format-code/src/textmate/index.ts index 0e692d5..d768294 100644 --- a/packages/neuri/src/formats/code/textmate.ts +++ b/packages/format-code/src/textmate/index.ts @@ -1,6 +1,7 @@ import type { LanguageRegistration, ShikiInternal } from '@shikijs/core' -import { createShikiInternal, loadWasm } from '@shikijs/core' import type { BundledLanguage } from 'shiki' + +import { createShikiInternal, loadWasm } from '@shikijs/core' import { bundledLanguages } from 'shiki' let wasmLoaded = false diff --git a/packages/neuri/package.json b/packages/neuri/package.json index 4ebb936..24b6d9c 100644 --- a/packages/neuri/package.json +++ b/packages/neuri/package.json @@ -27,18 +27,10 @@ "types": "./dist/index.d.ts", "import": "./dist/index.mjs" }, - "./agents/fs": { - "types": "./dist/agents/fs/index.d.ts", - "import": "./dist/agents/fs/index.mjs" - }, "./formats/json": { "types": "./dist/formats/json/index.d.ts", "import": "./dist/formats/json/index.mjs" }, - "./formats/code/textmate": { - "types": "./dist/formats/code/textmate.d.ts", - "import": "./dist/formats/code/textmate.mjs" - }, "./formats/markdown": { "types": "./dist/formats/markdown/index.d.ts", "import": "./dist/formats/markdown/index.mjs" @@ -64,19 +56,13 @@ }, "dependencies": { "@guiiai/logg": "^1.0.3", - "@shikijs/core": "^1.14.1", - "@shikijs/vscode-textmate": "^9.2.2", "@valibot/to-json-schema": "^0.2.1", "ajv": "^8.17.1", "defu": "^6.1.4", - "execa": "^9.3.1", - "ignore": "^5.3.2", "json-schema": "^0.4.0", "nanoid": "^5.0.7", "openai": "^4.54.0", "remark-parse": "^11.0.0", - "shiki": "^1.14.1", - "tm-grammars": "^1.17.18", "unified": "^11.0.5", "unist-util-filter": "^5.0.1", "unist-util-is": "^6.0.0", diff --git a/packages/neuri/src/formats/json/jsonParser.test.ts b/packages/neuri/src/formats/json/jsonParser.test.ts index c03a54a..2552c60 100644 --- a/packages/neuri/src/formats/json/jsonParser.test.ts +++ b/packages/neuri/src/formats/json/jsonParser.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest' import type { Token } from './jsonParser' +import { describe, expect, it } from 'vitest' import { createJSONParser, stringifyTokens } from './jsonParser' describe('jsonParser', () => { diff --git a/packages/neuri/src/formats/json/jsonParser.ts b/packages/neuri/src/formats/json/jsonParser.ts index cafe643..e39d21d 100644 --- a/packages/neuri/src/formats/json/jsonParser.ts +++ b/packages/neuri/src/formats/json/jsonParser.ts @@ -56,7 +56,7 @@ export function createJSONParser() { } function handleStateJSONNumber(char: string) { - if (/[\d.eE+-]/.test(char)) { + if (/[\d.e+-]/i.test(char)) { buffer += char } else { @@ -101,7 +101,7 @@ export function createJSONParser() { } function trimTrailingNonNumber(s: string): string { - return s.replace(/[^0-9.eE+-]+$/, '') + return s.replace(/[^0-9.e+-]+$/i, '') } function determineValueType(value: string): TokenType { @@ -109,7 +109,8 @@ export function createJSONParser() { return 'JSONBoolean' if (value === 'null') return 'JSONNull' - if (/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(value)) + // eslint-disable-next-line regexp/no-unused-capturing-group + if (/^-?\d+(\.\d+)?(e[+-]?\d+)?$/i.test(value)) return 'JSONNumber' return 'JSONString' } diff --git a/packages/neuri/src/formats/json/jsonschema.test.ts b/packages/neuri/src/formats/json/jsonschema.test.ts index 35db0d9..2c2b672 100644 --- a/packages/neuri/src/formats/json/jsonschema.test.ts +++ b/packages/neuri/src/formats/json/jsonschema.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest' import type { JSONSchemaDraft202012 } from './jsonschema' +import { describe, expect, it } from 'vitest' import { buildRegexFromSchema, buildRegexFromSchemaString, extractBySchema, extractObjectBySchema } from './jsonschema' describe('buildRegexFromSchemaString', () => { diff --git a/packages/neuri/src/formats/json/streamJsonParser.ts b/packages/neuri/src/formats/json/streamJsonParser.ts index 869df4d..69b27c4 100644 --- a/packages/neuri/src/formats/json/streamJsonParser.ts +++ b/packages/neuri/src/formats/json/streamJsonParser.ts @@ -1,6 +1,6 @@ import type { JSONSchema7 } from 'json-schema' -import Ajv from 'ajv' import type { Token } from './jsonParser' +import Ajv from 'ajv' import { createJSONParser } from './jsonParser' type JSONValue = string | number | boolean | null | { [key: string]: JSONValue } | JSONValue[] diff --git a/packages/neuri/src/formats/markdown/markdown.ts b/packages/neuri/src/formats/markdown/markdown.ts index 470f6e4..ebed170 100644 --- a/packages/neuri/src/formats/markdown/markdown.ts +++ b/packages/neuri/src/formats/markdown/markdown.ts @@ -1,6 +1,6 @@ -import { unified } from 'unified' -import RemarkParse from 'remark-parse' import type { Code, Node } from 'mdast' +import RemarkParse from 'remark-parse' +import { unified } from 'unified' import { visit } from 'unist-util-visit' function isCodeNode(node: Node): node is Code { @@ -46,7 +46,7 @@ interface StreamableParser { } function trimCodeBlockBoundary(content: string): string { - return content.replace(/\n(`){1,}$/, '') + return content.replace(/\n(`)+$/, '') } /** diff --git a/packages/use-fs/package.json b/packages/use-fs/package.json new file mode 100644 index 0000000..6fe905c --- /dev/null +++ b/packages/use-fs/package.json @@ -0,0 +1,55 @@ +{ + "name": "@neuri/use-fs", + "type": "module", + "version": "0.0.5", + "description": "FileSystem related agent implementation for neuri", + "author": { + "name": "Lingtic.io Neuri Team", + "email": "neuridev@lingtic.io", + "url": "https://github.com/lingticio" + }, + "license": "MIT", + "homepage": "https://docs.lingtic.io/pages/en/neuri", + "repository": { + "type": "git", + "url": "https://github.com/lingticio/neuri-js.git", + "directory": "packages/use-fs" + }, + "keywords": [ + "guiiai", + "llm", + "openai", + "lingtic", + "neuri" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs" + } + }, + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "files": [ + "LICENSE", + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "unbuild --stub", + "stub": "unbuild --stub", + "build": "unbuild", + "package:publish": "pnpm build && pnpm publish --access public --no-git-checks" + }, + "dependencies": { + "@guiiai/logg": "^1.0.3", + "defu": "^6.1.4", + "execa": "^9.3.1", + "ignore": "^5.3.2", + "neuri": "workspace:^" + }, + "devDependencies": { + "@types/node": "^20.11.17" + } +} diff --git a/packages/neuri/src/agents/fs/index.test.ts b/packages/use-fs/src/index.test.ts similarity index 100% rename from packages/neuri/src/agents/fs/index.test.ts rename to packages/use-fs/src/index.test.ts index a764566..495322f 100644 --- a/packages/neuri/src/agents/fs/index.test.ts +++ b/packages/use-fs/src/index.test.ts @@ -1,11 +1,11 @@ -import { fileURLToPath } from 'node:url' -import { dirname, join } from 'node:path' import { readFile as fsReadFile, writeFile as fsWriteFile, mkdir, rmdir } from 'node:fs/promises' -import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import { Format, LogLevel, setGlobalFormat, setGlobalLogLevel } from '@guiiai/logg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { exists } from './utils' import { FileSystem } from '.' +import { exists } from './utils' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) diff --git a/packages/neuri/src/agents/fs/index.ts b/packages/use-fs/src/index.ts similarity index 98% rename from packages/neuri/src/agents/fs/index.ts rename to packages/use-fs/src/index.ts index 2320d3f..2f497d6 100644 --- a/packages/neuri/src/agents/fs/index.ts +++ b/packages/use-fs/src/index.ts @@ -1,17 +1,17 @@ +import type { CallableComponent } from 'neuri' import { readFile as fsReadFile, writeFile as fsWriteFile, readdir, } from 'node:fs/promises' -import { cwd } from 'node:process' import { join, relative } from 'node:path' -import { defu } from 'defu' +import { cwd } from 'node:process' import { useLogg } from '@guiiai/logg' -import ignore from 'ignore' +import { defu } from 'defu' import { execa } from 'execa' -import type { CallableComponent } from '../../callable' -import { defineCallable, defineCallableComponent } from '../../callable' +import ignore from 'ignore' +import { defineCallable, defineCallableComponent } from 'neuri' import { exists } from './utils' export function FileSystem(options?: { basePath?: string }): CallableComponent { diff --git a/packages/neuri/src/agents/fs/utils.ts b/packages/use-fs/src/utils.ts similarity index 100% rename from packages/neuri/src/agents/fs/utils.ts rename to packages/use-fs/src/utils.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08d28a0..43f8bf3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -95,7 +95,7 @@ importers: specifier: ^4.19.1 version: 4.19.1 - packages/neuri: + packages/format-code: dependencies: '@guiiai/logg': specifier: ^1.0.3 @@ -106,6 +106,22 @@ importers: '@shikijs/vscode-textmate': specifier: ^9.2.2 version: 9.2.2 + shiki: + specifier: ^1.14.1 + version: 1.14.1 + tm-grammars: + specifier: ^1.17.18 + version: 1.17.18 + devDependencies: + '@types/node': + specifier: ^20.11.17 + version: 20.16.12 + + packages/neuri: + dependencies: + '@guiiai/logg': + specifier: ^1.0.3 + version: 1.0.3 '@valibot/to-json-schema': specifier: ^0.2.1 version: 0.2.1(valibot@0.42.1(typescript@5.6.3)) @@ -115,12 +131,6 @@ importers: defu: specifier: ^6.1.4 version: 6.1.4 - execa: - specifier: ^9.3.1 - version: 9.3.1 - ignore: - specifier: ^5.3.2 - version: 5.3.2 json-schema: specifier: ^0.4.0 version: 0.4.0 @@ -133,12 +143,6 @@ importers: remark-parse: specifier: ^11.0.0 version: 11.0.0 - shiki: - specifier: ^1.14.1 - version: 1.14.1 - tm-grammars: - specifier: ^1.17.18 - version: 1.17.18 unified: specifier: ^11.0.5 version: 11.0.5 @@ -177,6 +181,28 @@ importers: specifier: ^15.0.1 version: 15.0.1 + packages/use-fs: + dependencies: + '@guiiai/logg': + specifier: ^1.0.3 + version: 1.0.3 + defu: + specifier: ^6.1.4 + version: 6.1.4 + execa: + specifier: ^9.3.1 + version: 9.3.1 + ignore: + specifier: ^5.3.2 + version: 5.3.2 + neuri: + specifier: workspace:^ + version: link:../neuri + devDependencies: + '@types/node': + specifier: ^20.11.17 + version: 20.16.12 + packages: '@algolia/autocomplete-core@1.9.3': @@ -5753,7 +5779,7 @@ snapshots: '@types/node-fetch@2.6.11': dependencies: - '@types/node': 20.16.12 + '@types/node': 22.7.6 form-data: 4.0.0 '@types/node@18.19.43':