diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..f8bea77 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,127 @@ +name: test + +on: + push: + branches: [main] + pull_request: + +jobs: + unit: + name: typecheck / lint / unit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + # src/version.ts 是 prebuild 生成的，typecheck 之前必须先有 + - run: node scripts/generate-version.js + - run: pnpm exec tsc --noEmit + # 阻塞门槛只看存储层：仓库里既有的 src/ 文件本来就有 62 个 lint 错误 + # （多为 no-explicit-any 和风格问题），不该由这个 PR 顺带清理，也不该 + # 让每个后续 PR 都被它挡住 + - run: pnpm exec eslint src/storage --ext .ts + # 全量 lint 只做提示，等既有问题清完再改成阻塞 + - run: pnpm lint + continue-on-error: true + # 同理，阻塞门槛只看存储层的测试。仓库里有 4 个测试套件因为 + # adapters -> core -> index 的既有循环依赖在 CJS 下加载不了， + # 这个 PR 之前 jest 连启动都启动不了，现在能跑 55 个用例， + # 剩下那 4 个套件要单独拆环或把 jest 切到 ESM 才能修 + - run: pnpm exec jest src/storage + # 全量测试同样只做提示 + - run: pnpm test + continue-on-error: true + + pglite: + name: postgres semantics (PGlite) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: node scripts/generate-version.js + # PGlite 是编译成 WASM 的 Postgres，用来验证生成的 SQL 在 Postgres 上确实 + # 跑得通；不需要服务端，所以比下面那个 job 快得多 + - run: pnpm test:pg + + postgres: + name: postgres integration (real server) + runs-on: ubuntu-latest + services: + postgres: + # 用 pgvector 镜像：记忆子系统后续要用到，顺带把扩展可用性也固定下来 + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: chaite + POSTGRES_DB: chaite_test + # 用 trust 而不是写一个密码进仓库：这个容器只在 job 生命周期内存在、 + # 只监听 runner 本地，没有任何需要保护的东西。顺带也就不会有一个 + # 看起来像凭据的字符串躺在 git 历史里让密钥扫描器报警。 + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U chaite -d chaite_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: node scripts/generate-version.js + # 这一组测的是 PGlite 覆盖不到的东西：TCP、认证、连接池超时与上限、 + # 多 client 并发事务、pool.end()、以及 SQLite -> Postgres 整体迁移 + - run: pnpm test:pg:integration + env: + CHAITE_PG_HOST: 127.0.0.1 + CHAITE_PG_PORT: 5432 + CHAITE_PG_DB: chaite_test + CHAITE_PG_USER: chaite + + build: + name: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + # 确认打出来的包真的能被 import，且存储层的导出都在 + - name: smoke test the built package + run: | + node -e " + import('./dist/index.mjs').then(m => { + const want = ['SqlKvStorage','SqliteDriver','PostgresDriver','DriverRegistry','drivers', + 'SqlChannelStorage','SqlChatPresetStorage','SqlToolsStorage','SqlTriggerStorage', + 'SqlProcessorsStorage','SqlToolsGroupStorage','SqlUserStateStorage','SqlMcpServerStorage', + 'SqlOperationLogStorage','SqlHistoryManager','migrateSqlStorage','getChaiteTables'] + const missing = want.filter(n => !(n in m)) + if (missing.length) { console.error('missing exports: ' + missing.join(', ')); process.exit(1) } + console.log('all ' + want.length + ' storage exports present') + }).catch(e => { console.error(e); process.exit(1) }) + " diff --git a/eslint.config.mjs b/eslint.config.mjs index 10b9d57..5519491 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -3,7 +3,13 @@ import tseslint from 'typescript-eslint' export default [ { - ignores: ['dist/**', 'node_modules/**', '**/*.test.ts', 'frontend/**/*', 'tsdown.config.ts'] + ignores: [ + 'dist/**', 'node_modules/**', '**/*.test.ts', 'frontend/**/*', 'tsdown.config.ts', + // 以下都不是 TS 源码：构建脚本、CJS 配置、测试替身，以及 prebuild 生成的 + // version.ts。它们跑的是 Node CJS/脚本环境，按 TS 源码的规则去检只会得到 + // 一堆和代码质量无关的 no-undef + 'scripts/**', '**/*.cjs', 'test/**', 'src/version.ts', + ], }, eslint.configs.recommended, ...tseslint.configs.recommended, @@ -18,6 +24,10 @@ export default [ rules: { "no-unused-vars": 0, "@typescript-eslint/no-unused-vars": 0, + // TS 编译器本身就会报未定义变量，而且它认得 lib/@types 里的全局声明。 + // eslint 的 no-undef 在 TS 上只会误报（console/process 之类）， + // typescript-eslint 官方也建议关掉。 + 'no-undef': 0, // 控制大括号内容前后有空格 'object-curly-spacing': ['error', 'always'], diff --git a/jest.config.cjs b/jest.config.cjs new file mode 100644 index 0000000..b5a3a12 --- /dev/null +++ b/jest.config.cjs @@ -0,0 +1,49 @@ +// jest.config.js +module.exports = { + // 直接使用 ts-jest 的默认预设，它对大部分 TS 项目都适用 + preset: 'ts-jest', + // 测试环境依然是 node + testEnvironment: 'node', + moduleNameMapper: { + // 这条规则是关键，它使用了正则表达式 + // ^src/(.*)$ : 匹配所有以 'src/' 开头的导入路径 + // /src/$1 : 将它替换成一个完整的物理路径 + '^src/(.*)$': '/src/$1', + + // @karinjs/node-schedule 只发 ESM，jest 在 CJS 模式下加载不了 .mjs。 + // types/trigger.ts 在顶层 import 它，于是所有 import 到 types 桶文件的 + // 测试都会被连累。测试里换成最小替身，见该文件里的说明。 + '^@karinjs/node-schedule$': '/test/stubs/node-schedule.cjs', + + // src/__dirname.ts 用 import.meta.url 推导路径，在 CJS 下编译不了。 + // CJS 本身就有 __dirname，测试里换成等价实现。 + '^(.*)/__dirname$': '/test/stubs/dirname.cjs', + }, + // Postgres 那套要 PGlite（WASM），而 PGlite 的动态 import 需要 + // --experimental-vm-modules。默认跑排除掉，用 pnpm test:pg 单独跑。 + testPathIgnorePatterns: ['/node_modules/', '/frontend/', 'postgres\\.test\\.ts$', 'migrate\\.pg\\.test\\.ts$', 'postgres\\.integration\\.test\\.ts$'], + transformIgnorePatterns: [ + // 默认 Jest 会忽略掉所有 node_modules 里的文件 + // 我们需要修改这个默认行为，允许 Jest 转换 @karinjs/node-schedule + // + // 这里不能只写 (?!@karinjs/node-schedule/)：pnpm 的真实路径是 + // node_modules/.pnpm/@karinjs+node-schedule@x.y.z/node_modules/@karinjs/node-schedule/…， + // 否定环视卡在第一个 node_modules/ 后面，看到的是 .pnpm/ 而不是包名， + // 于是这个包又被排除掉了。改成匹配整条路径里是否出现过这个包名， + // npm/yarn 的扁平布局和 pnpm 的嵌套布局都能覆盖。 + '/node_modules/(?!.*@karinjs[+/]node-schedule).+\\.(js|jsx|mjs|cjs|ts|tsx)$' + ], + transform: { + // 遇到 .ts/.tsx 文件，使用 ts-jest 来转换 + '^.+\\.tsx?$': 'ts-jest', + + // .js/.jsx/.mjs 也交给 ts-jest：@karinjs/node-schedule 发的是 ESM。 + // 必须显式打开 allowJs，否则 ts-jest 会直接把这些文件原样放过去， + // 结果就是在 CJS 环境里执行到 import 语句而报 + // "Cannot use import statement outside a module"。 + // 只在测试里开，不动 tsconfig.json，免得影响构建。 + // module 也要显式写成 commonjs：项目 tsconfig 是 ES2022，而 jest 这里 + // 跑的是 CJS，一旦传了内联 tsconfig，ts-jest 就不再自动改写 module 了。 + '^.+\\.(jsx?|mjs)$': ['ts-jest', { tsconfig: { allowJs: true, module: 'commonjs' } }], + }, +}; diff --git a/jest.config.js b/jest.config.js deleted file mode 100644 index 9ac542a..0000000 --- a/jest.config.js +++ /dev/null @@ -1,29 +0,0 @@ -// jest.config.js -module.exports = { - // 直接使用 ts-jest 的默认预设，它对大部分 TS 项目都适用 - preset: 'ts-jest', - // 测试环境依然是 node - testEnvironment: 'node', - moduleNameMapper: { - // 这条规则是关键，它使用了正则表达式 - // ^src/(.*)$ : 匹配所有以 'src/' 开头的导入路径 - // /src/$1 : 将它替换成一个完整的物理路径 - '^src/(.*)$': '/src/$1', - }, - transformIgnorePatterns: [ - // 默认 Jest 会忽略掉所有 node_modules 里的文件 - // 我们需要修改这个默认行为，允许 Jest 转换 @karinjs/node-schedule - '/node_modules/(?!@karinjs/node-schedule/).+\\.(js|jsx|mjs|cjs|ts|tsx)$' - ], - transform: { - // 遇到 .ts/.tsx 文件，使用 ts-jest 来转换 - '^.+\\.tsx?$': 'ts-jest', - - // 遇到 .js/.jsx 文件，也使用 ts-jest 来转换 - // 这是解决您问题的关键，因为 @karinjs/node-schedule 是 .js 文件 - '^.+\\.jsx?$': 'ts-jest', - - // 有些 ESM 包可能使用 .mjs 后缀，以防万一也加上 - '^.+\\.mjs$': 'ts-jest', - }, -}; diff --git a/package.json b/package.json index 19296a4..836da34 100644 --- a/package.json +++ b/package.json @@ -28,10 +28,14 @@ "prebuild": "node scripts/generate-version.js", "prepublishOnly": "npm run build", "pub": "npm publish --access public", - "test": "jest" + "test": "jest", + "test:pg": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=/node_modules/ --testPathPattern=\"(postgres|migrate\\.pg)\\.test\\.ts\"", + "test:pg:integration": "jest --testPathIgnorePatterns=/node_modules/ --testPathPattern=\"postgres\\.integration\\.test\\.ts\" --runInBand" }, "devDependencies": { "@anthropic-ai/sdk": "^0.39.0", + "@electric-sql/pglite": "^0.5.8", + "@electric-sql/pglite-pgvector": "^0.0.9", "@eslint/js": "^9.21.0", "@google/genai": "^2.19.0", "@karinjs/node-schedule": "^1.1.2", @@ -40,6 +44,7 @@ "@types/jest": "^29.5.14", "@types/jsonwebtoken": "^9.0.9", "@types/node": "^20.17.24", + "@types/pg": "^8.23.1", "@typescript-eslint/eslint-plugin": "^8.25.0", "@typescript-eslint/parser": "^8.25.0", "axios": "^1.8.1", @@ -55,23 +60,38 @@ "mammoth": "^1.9.0", "openai": "^5.20.1", "pdf-parse": "^2.4.5", + "pg": "^8.23.0", "prettier": "^3.5.2", "reflect-metadata": "^0.2.2", + "sqlite3": "^6.0.1", "ts-jest": "^29.2.6", "tsdown": "^0.16.0", "typescript": "^5.7.3", "typescript-eslint": "^8.25.0" }, "peerDependencies": { - "pdf-parse": "^2.4.5" + "pdf-parse": "^2.4.5", + "pg": "^8.0.0", + "sqlite3": ">=5 <7" }, "peerDependenciesMeta": { "pdf-parse": { "optional": true + }, + "pg": { + "optional": true + }, + "sqlite3": { + "optional": true } }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^3.25.76" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "sqlite3" + ] } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4b29cf6..8f21b9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,12 @@ importers: '@anthropic-ai/sdk': specifier: ^0.39.0 version: 0.39.0 + '@electric-sql/pglite': + specifier: ^0.5.8 + version: 0.5.8 + '@electric-sql/pglite-pgvector': + specifier: ^0.0.9 + version: 0.0.9(@electric-sql/pglite@0.5.8) '@eslint/js': specifier: ^9.21.0 version: 9.22.0 @@ -42,6 +48,9 @@ importers: '@types/node': specifier: ^20.17.24 version: 20.17.24 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 '@typescript-eslint/eslint-plugin': specifier: ^8.25.0 version: 8.26.1(@typescript-eslint/parser@8.26.1(eslint@9.22.0(jiti@2.6.1))(typescript@5.8.2))(eslint@9.22.0(jiti@2.6.1))(typescript@5.8.2) @@ -87,12 +96,18 @@ importers: pdf-parse: specifier: ^2.4.5 version: 2.4.5 + pg: + specifier: ^8.23.0 + version: 8.23.0 prettier: specifier: ^3.5.2 version: 3.5.3 reflect-metadata: specifier: ^0.2.2 version: 0.2.2 + sqlite3: + specifier: ^6.0.1 + version: 6.0.1 ts-jest: specifier: ^29.2.6 version: 29.2.6(@babel/core@7.26.10)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.10))(jest@29.7.0(@types/node@20.17.24))(typescript@5.8.2) @@ -297,6 +312,14 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + '@electric-sql/pglite-pgvector@0.0.9': + resolution: {integrity: sha512-ue4iBW651gDQwBwn97Ekv1lYGPvXa1ymHbRbTCSL0Ib286PRDD1VDOwzwEoekZuO/wctMbTzKzlwdgDwYrqZ8A==} + peerDependencies: + '@electric-sql/pglite': 0.5.8 + + '@electric-sql/pglite@0.5.8': + resolution: {integrity: sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q==} + '@emnapi/core@1.7.0': resolution: {integrity: sha512-pJdKGq/1iquWYtv1RRSljZklxHCOCAJFJrImO5ZLKPJVJlVUcs8yFwNQlqS0Lo8xT1VAXXTCZocF9n26FWEKsw==} @@ -383,6 +406,10 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': + resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} + engines: {node: '>=18.0.0'} + '@istanbuljs/load-nyc-config@1.1.0': resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -527,35 +554,30 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@napi-rs/canvas-linux-arm64-musl@0.1.80': resolution: {integrity: sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': resolution: {integrity: sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] - libc: [glibc] '@napi-rs/canvas-linux-x64-gnu@0.1.80': resolution: {integrity: sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@napi-rs/canvas-linux-x64-musl@0.1.80': resolution: {integrity: sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@napi-rs/canvas-win32-x64-msvc@0.1.80': resolution: {integrity: sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==} @@ -658,28 +680,24 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.0.0-beta.46': resolution: {integrity: sha512-6IZHycZetmVaC9zwcl1aA9fPYPuxLa5apALjJRoJu/2BZdER3zBWxDnCzlEh4SUlo++cwdfV9ZQRK9JS8cLNuA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-x64-gnu@1.0.0-beta.46': resolution: {integrity: sha512-R/kI8fMnsxXvWzcMv5A408hfvrwtAwD/HdQKIE1HKWmfxdSHB11Y3PVwlnt7RVo7I++6mWCIxxj5o3gut4ibEw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.0.0-beta.46': resolution: {integrity: sha512-vGUXKuHGUlG2XBwvN4A8KIegeaVVxN2ZxdGG9thycwRkzUvZ9ccKvqUVZM8cVRyNRWgVgsGCS18qLUefVplwKw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.0.0-beta.46': resolution: {integrity: sha512-6SpDGH+0Dud3/RFDoC6fva6+Cm/0COnMRKR8kI4ssHWlCXPymlM59kYFCIBLZZqwURpNVVMPln4rWjxXuwD23w==} @@ -794,6 +812,9 @@ packages: '@types/node@20.17.24': resolution: {integrity: sha512-d7fGCyB96w9BnWQrOsJtpyiSaBcAYYr75bnK6ZRjDbql2cGLj/3GsL5OYmLPNq76l7Gf2q4Rv9J2o6h5CrD9sA==} + '@types/pg@8.23.1': + resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + '@types/qs@6.9.18': resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==} @@ -870,6 +891,10 @@ packages: engines: {node: '>=10.0.0'} deprecated: this version has critical issues, please update to the latest version + abbrev@4.0.0: + resolution: {integrity: sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==} + engines: {node: ^20.17.0 || >=22.9.0} + abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -1002,9 +1027,15 @@ packages: bignumber.js@9.3.1: resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + birpc@2.7.0: resolution: {integrity: sha512-tub/wFGH49vNCm0xraykcY3TcRgX/3JsALYq/Lwrtti+bTyFHkCUAWF5wgYoie8P41wYwig2mIKiqoocr1EkEQ==} + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + bluebird@3.4.7: resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} @@ -1044,6 +1075,9 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1087,6 +1121,13 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} + ci-info@3.9.0: resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} engines: {node: '>=8'} @@ -1202,6 +1243,10 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dedent@1.5.3: resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==} peerDependencies: @@ -1210,6 +1255,10 @@ packages: babel-plugin-macros: optional: true + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1232,6 +1281,10 @@ packages: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + detect-newline@3.1.0: resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} engines: {node: '>=8'} @@ -1302,6 +1355,13 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + env-paths@2.2.1: + resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} + engines: {node: '>=6'} + error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} @@ -1428,10 +1488,17 @@ packages: resolution: {integrity: sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==} engines: {node: '>= 0.8.0'} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect@29.7.0: resolution: {integrity: sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + exponential-backoff@3.1.3: + resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} + express-rate-limit@8.5.2: resolution: {integrity: sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==} engines: {node: '>= 16'} @@ -1491,6 +1558,9 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} @@ -1561,6 +1631,9 @@ packages: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -1607,6 +1680,9 @@ packages: get-tsconfig@4.13.0: resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1707,6 +1783,9 @@ packages: resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} engines: {node: '>=0.10.0'} + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -1734,6 +1813,9 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ip-address@10.2.0: resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} engines: {node: '>= 12'} @@ -1785,6 +1867,10 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isexe@4.0.0: + resolution: {integrity: sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==} + engines: {node: '>=20'} + istanbul-lib-coverage@3.2.2: resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} engines: {node: '>=8'} @@ -2159,6 +2245,10 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -2170,10 +2260,20 @@ packages: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} + minizlib@3.1.0: + resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} + engines: {node: '>= 18'} + + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mkdirp@1.0.4: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} @@ -2185,6 +2285,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -2196,6 +2299,14 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-abi@3.96.0: + resolution: {integrity: sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==} + engines: {node: '>=10'} + + node-addon-api@8.9.2: + resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2214,6 +2325,11 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp@12.4.0: + resolution: {integrity: sha512-OMcPNvqTCFUnNaBlmdgq+lfNqY7gTiSmNRDjY3uAXRyudeKZEZxu3CLtjMQrx4zZxCX2b/mpNqTtwuCJgXhHkw==} + engines: {node: ^20.17.0 || >=22.9.0} + hasBin: true + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -2223,6 +2339,11 @@ packages: noms@0.0.0: resolution: {integrity: sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==} + nopt@9.0.0: + resolution: {integrity: sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==} + engines: {node: ^20.17.0 || >=22.9.0} + hasBin: true + normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -2348,6 +2469,40 @@ packages: resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} engines: {node: '>=20.16.0 || >=22.3.0'} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.16.0: + resolution: {integrity: sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.23.0: + resolution: {integrity: sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -2371,6 +2526,28 @@ packages: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -2388,6 +2565,10 @@ packages: resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + proc-log@6.1.0: + resolution: {integrity: sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==} + engines: {node: ^20.17.0 || >=22.9.0} + process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} @@ -2406,6 +2587,9 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -2439,6 +2623,10 @@ packages: resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} engines: {node: '>= 0.10'} + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} @@ -2448,6 +2636,10 @@ packages: readable-stream@2.3.8: resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -2614,6 +2806,12 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -2628,9 +2826,17 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sqlite3@6.0.1: + resolution: {integrity: sha512-X0czUUMG2tmSqJpEQa3tCuZSHKIx8PwM53vLZzKp/o6Rpy25fiVfjdbnZ988M8+O3ZWR1ih0K255VumCb3MAnQ==} + engines: {node: '>=20.17.0'} + stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -2677,6 +2883,10 @@ packages: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -2697,6 +2907,17 @@ packages: resolution: {integrity: sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==} engines: {node: ^14.18.0 || >=16.0.0} + tar-fs@2.1.5: + resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + + tar@7.5.22: + resolution: {integrity: sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==} + engines: {node: '>=18'} + test-exclude@6.0.0: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} @@ -2791,6 +3012,9 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -2835,6 +3059,10 @@ packages: undici-types@6.19.8: resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + undici@6.28.1: + resolution: {integrity: sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==} + engines: {node: '>=18.17'} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -2889,6 +3117,11 @@ packages: engines: {node: '>= 8'} hasBin: true + which@6.0.1: + resolution: {integrity: sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==} + engines: {node: ^20.17.0 || >=22.9.0} + hasBin: true + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} @@ -2935,6 +3168,10 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yallist@5.0.0: + resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} + engines: {node: '>=18'} + yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} @@ -3190,6 +3427,12 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} + '@electric-sql/pglite-pgvector@0.0.9(@electric-sql/pglite@0.5.8)': + dependencies: + '@electric-sql/pglite': 0.5.8 + + '@electric-sql/pglite@0.5.8': {} + '@emnapi/core@1.7.0': dependencies: '@emnapi/wasi-threads': 1.1.0 @@ -3289,6 +3532,10 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': + dependencies: + minipass: 7.1.2 + '@istanbuljs/load-nyc-config@1.1.0': dependencies: camelcase: 5.3.1 @@ -3763,6 +4010,12 @@ snapshots: dependencies: undici-types: 6.19.8 + '@types/pg@8.23.1': + dependencies: + '@types/node': 20.17.24 + pg-protocol: 1.16.0 + pg-types: 2.2.0 + '@types/qs@6.9.18': {} '@types/range-parser@1.2.7': {} @@ -3867,6 +4120,9 @@ snapshots: '@xmldom/xmldom@0.8.10': {} + abbrev@4.0.0: + optional: true + abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -4020,8 +4276,18 @@ snapshots: bignumber.js@9.3.1: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + birpc@2.7.0: {} + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + bluebird@3.4.7: {} body-parser@1.20.3: @@ -4087,6 +4353,11 @@ snapshots: buffer-from@1.1.2: {} + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bytes@3.1.2: {} cac@6.7.14: {} @@ -4120,6 +4391,10 @@ snapshots: dependencies: readdirp: 4.1.2 + chownr@1.1.4: {} + + chownr@3.0.0: {} + ci-info@3.9.0: {} cjs-module-lexer@1.4.3: {} @@ -4222,8 +4497,14 @@ snapshots: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + dedent@1.5.3: {} + deep-extend@0.6.0: {} + deep-is@0.1.4: {} deepmerge@4.3.1: {} @@ -4236,6 +4517,8 @@ snapshots: destroy@1.2.0: {} + detect-libc@2.1.2: {} + detect-newline@3.1.0: {} diff-sequences@29.6.3: {} @@ -4282,6 +4565,13 @@ snapshots: encodeurl@2.0.0: {} + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + env-paths@2.2.1: + optional: true + error-ex@1.3.2: dependencies: is-arrayish: 0.2.1 @@ -4417,6 +4707,8 @@ snapshots: exit@0.1.2: {} + expand-template@2.0.3: {} + expect@29.7.0: dependencies: '@jest/expect-utils': 29.7.0 @@ -4425,6 +4717,9 @@ snapshots: jest-message-util: 29.7.0 jest-util: 29.7.0 + exponential-backoff@3.1.3: + optional: true + express-rate-limit@8.5.2(express@5.2.1): dependencies: express: 5.2.1 @@ -4540,6 +4835,8 @@ snapshots: dependencies: flat-cache: 4.0.1 + file-uri-to-path@1.0.0: {} + filelist@1.0.4: dependencies: minimatch: 5.1.6 @@ -4619,6 +4916,8 @@ snapshots: fresh@2.0.0: {} + fs-constants@1.0.0: {} + fs.realpath@1.0.0: {} fsevents@2.3.3: @@ -4673,6 +4972,8 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -4785,6 +5086,8 @@ snapshots: dependencies: safer-buffer: 2.1.2 + ieee754@1.2.1: {} + ignore@5.3.2: {} immediate@3.0.6: {} @@ -4808,6 +5111,8 @@ snapshots: inherits@2.0.4: {} + ini@1.3.8: {} + ip-address@10.2.0: {} ipaddr.js@1.9.1: {} @@ -4840,6 +5145,9 @@ snapshots: isexe@2.0.0: {} + isexe@4.0.0: + optional: true + istanbul-lib-coverage@3.2.2: {} istanbul-lib-instrument@5.2.1: @@ -5402,6 +5710,8 @@ snapshots: mimic-fn@2.1.0: {} + mimic-response@3.1.0: {} + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 @@ -5414,20 +5724,36 @@ snapshots: dependencies: brace-expansion: 2.0.1 + minimist@1.2.8: {} + minipass@7.1.2: {} + minizlib@3.1.0: + dependencies: + minipass: 7.1.2 + + mkdirp-classic@0.5.3: {} + mkdirp@1.0.4: {} ms@2.0.0: {} ms@2.1.3: {} + napi-build-utils@2.0.0: {} + natural-compare@1.4.0: {} negotiator@0.6.3: {} negotiator@1.0.0: {} + node-abi@3.96.0: + dependencies: + semver: 7.7.3 + + node-addon-api@8.9.2: {} + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -5440,6 +5766,20 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-gyp@12.4.0: + dependencies: + env-paths: 2.2.1 + exponential-backoff: 3.1.3 + graceful-fs: 4.2.11 + nopt: 9.0.0 + proc-log: 6.1.0 + semver: 7.7.3 + tar: 7.5.22 + tinyglobby: 0.2.15 + undici: 6.28.1 + which: 6.0.1 + optional: true + node-int64@0.4.0: {} node-releases@2.0.19: {} @@ -5449,6 +5789,11 @@ snapshots: inherits: 2.0.4 readable-stream: 1.0.34 + nopt@9.0.0: + dependencies: + abbrev: 4.0.0 + optional: true + normalize-path@3.0.0: {} npm-run-path@4.0.1: @@ -5555,6 +5900,41 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.80 + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.23.0): + dependencies: + pg: 8.23.0 + + pg-protocol@1.16.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.23.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.23.0) + pg-protocol: 1.16.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -5569,6 +5949,31 @@ snapshots: dependencies: find-up: 4.1.0 + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.96.0 + pump: 3.0.4 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.5 + tunnel-agent: 0.6.0 + prelude-ls@1.2.1: {} prettier-linter-helpers@1.0.0: @@ -5583,6 +5988,9 @@ snapshots: ansi-styles: 5.2.0 react-is: 18.3.1 + proc-log@6.1.0: + optional: true + process-nextick-args@2.0.1: {} prompts@2.4.2: @@ -5611,6 +6019,11 @@ snapshots: proxy-from-env@1.1.0: {} + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + punycode@2.3.1: {} pure-rand@6.1.0: {} @@ -5644,6 +6057,13 @@ snapshots: iconv-lite: 0.7.3 unpipe: 1.0.0 + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + react-is@18.3.1: {} readable-stream@1.0.34: @@ -5663,6 +6083,12 @@ snapshots: string_decoder: 1.1.1 util-deprecate: 1.0.2 + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + readdirp@4.1.2: {} reflect-metadata@0.2.2: {} @@ -5868,6 +6294,14 @@ snapshots: signal-exit@4.1.0: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + sisteransi@1.0.5: {} slash@3.0.0: {} @@ -5879,8 +6313,19 @@ snapshots: source-map@0.6.1: {} + split2@4.2.0: {} + sprintf-js@1.0.3: {} + sqlite3@6.0.1: + dependencies: + bindings: 1.5.0 + node-addon-api: 8.9.2 + prebuild-install: 7.1.3 + tar: 7.5.22 + optionalDependencies: + node-gyp: 12.4.0 + stack-utils@2.0.6: dependencies: escape-string-regexp: 2.0.0 @@ -5924,6 +6369,8 @@ snapshots: strip-final-newline@2.0.0: {} + strip-json-comments@2.0.1: {} + strip-json-comments@3.1.1: {} supports-color@7.2.0: @@ -5941,6 +6388,29 @@ snapshots: '@pkgr/core': 0.1.1 tslib: 2.8.1 + tar-fs@2.1.5: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + + tar@7.5.22: + dependencies: + '@isaacs/fs-minipass': 4.0.1 + chownr: 3.0.0 + minipass: 7.1.2 + minizlib: 3.1.0 + yallist: 5.0.0 + test-exclude@6.0.0: dependencies: '@istanbuljs/schema': 0.1.3 @@ -6021,6 +6491,10 @@ snapshots: tslib@2.8.1: {} + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -6065,6 +6539,9 @@ snapshots: undici-types@6.19.8: {} + undici@6.28.1: + optional: true + unpipe@1.0.0: {} untildify@4.0.0: {} @@ -6110,6 +6587,11 @@ snapshots: dependencies: isexe: 2.0.0 + which@6.0.1: + dependencies: + isexe: 4.0.0 + optional: true + word-wrap@1.2.5: {} wrap-ansi@7.0.0: @@ -6141,6 +6623,8 @@ snapshots: yallist@3.1.1: {} + yallist@5.0.0: {} + yargs-parser@20.2.9: {} yargs-parser@21.1.1: {} diff --git a/src/index.ts b/src/index.ts index 126b6f2..5e8ebff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,14 @@ export * from './rag/index' export * from './channels/index' export * from './const/index' export * from './share/index' +export * from './storage/index' + +// 把 chaite 的 getLogger 注入存储层。存储层只做类型引用以避免把 types 桶文件 +// （以及它带来的 node-schedule / PdfParser）拉进自己的模块图，所以这里在包入口 +// 补上真正的实现。 +import { setStorageLoggerResolver } from './storage/logger' +import { getLogger as _getLogger } from './utils/logger' +setStorageLoggerResolver(_getLogger) export * from './version' export * from './core' export * from './controllers' diff --git a/src/storage/driver/dialect.ts b/src/storage/driver/dialect.ts new file mode 100644 index 0000000..b72d72d --- /dev/null +++ b/src/storage/driver/dialect.ts @@ -0,0 +1,135 @@ +import { ColumnSpec, ColumnTypeName, Dialect, DialectName, IndexColumn, IndexOptions } from './types' + +/** + * 方言差异。 + * + * 这些 storage 的表在关系意义上都不是关系数据：一个 TEXT 主键、几个为了能下推 + * 过滤而提升出来的列、剩下的塞进 JSON 列，没有任何 JOIN。所以真正随数据库变化 + * 的只有三件事：占位符写法、upsert 语法、建表时的类型名。收敛掉这三样，其余 + * SQL 就能共用一份。 + */ + +const TYPES: Record> = { + sqlite: { + text: 'TEXT', + int: 'INTEGER', + bigint: 'INTEGER', + // SQLite 没有布尔类型，历史数据里存的就是 0/1。Postgres 这边也继续用整数， + // 免得同一份 toRecord/fromRecord 要为两种表示分叉。 + bool: 'INTEGER', + json: 'TEXT', + blob: 'BLOB', + }, + postgres: { + text: 'TEXT', + int: 'INTEGER', + bigint: 'BIGINT', + bool: 'INTEGER', + json: 'TEXT', + blob: 'BYTEA', + }, +} + +class SqlDialect implements Dialect { + readonly name: DialectName + private readonly types: Record + + constructor(name: DialectName) { + const types = TYPES[name] + if (!types) { + throw new Error(`unknown dialect: ${name}`) + } + this.name = name + this.types = types + } + + get isSqlite(): boolean { + return this.name === 'sqlite' + } + + get isPostgres(): boolean { + return this.name === 'postgres' + } + + columnType(logicalType: ColumnTypeName): string { + const type = this.types[logicalType] + if (!type) { + throw new Error(`unknown column type: ${logicalType}`) + } + return type + } + + /** 两种方言都用双引号；保留这层是为了以后加 MySQL（反引号）。 */ + quoteId(name: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + throw new Error(`unsafe identifier: ${name}`) + } + return `"${name}"` + } + + autoIncrementPk(): string { + return this.isPostgres ? 'BIGSERIAL PRIMARY KEY' : 'INTEGER PRIMARY KEY AUTOINCREMENT' + } + + /** + * 两种方言的 ON CONFLICT 写法一致，差别只在 SQLite 还额外允许 + * INSERT OR REPLACE。统一成 ON CONFLICT，共用一份。 + * + * @param immutable 冲突时不覆盖的列，例如代理主键 + */ + upsert(table: string, columns: string[], conflictColumn = 'id', immutable: string[] = []): string { + const cols = columns.map(c => this.quoteId(c)).join(', ') + const placeholders = columns.map(() => '?').join(', ') + const updates = columns + .filter(c => c !== conflictColumn && !immutable.includes(c)) + .map(c => `${this.quoteId(c)} = EXCLUDED.${this.quoteId(c)}`) + .join(', ') + const quotedTable = this.quoteId(table) + const conflict = this.quoteId(conflictColumn) + // 只有主键一列时没有可更新的字段，退化成 DO NOTHING + if (!updates) { + return `INSERT INTO ${quotedTable} (${cols}) VALUES (${placeholders}) ON CONFLICT(${conflict}) DO NOTHING` + } + return `INSERT INTO ${quotedTable} (${cols}) VALUES (${placeholders}) ON CONFLICT(${conflict}) DO UPDATE SET ${updates}` + } + + createTable(table: string, columns: Record): string { + const defs = Object.entries(columns).map(([name, spec]) => { + const opts: ColumnSpec = typeof spec === 'string' ? { type: spec } : spec + let def = `${this.quoteId(name)} ${ + opts.autoIncrement ? this.autoIncrementPk() : this.columnType(opts.type ?? 'text') + }` + if (opts.pk && !opts.autoIncrement) def += ' PRIMARY KEY' + if (opts.notNull) def += ' NOT NULL' + if (opts.default !== undefined) def += ` DEFAULT ${formatDefault(opts.default)}` + return def + }) + return `CREATE TABLE IF NOT EXISTS ${this.quoteId(table)} (${defs.join(', ')})` + } + + createIndex(table: string, columns: IndexColumn[], options: IndexOptions = {}): string { + const { name, unique = false, order } = options + const names = columns.map(c => (typeof c === 'string' ? c : c.column)) + const indexName = name || `idx_${table}_${names.join('_')}` + const cols = columns + .map(c => { + const column = typeof c === 'string' ? c : c.column + const columnOrder = typeof c === 'string' ? order : c.order ?? order + return `${this.quoteId(column)}${columnOrder ? ` ${columnOrder}` : ''}` + }) + .join(', ') + return `CREATE${unique ? ' UNIQUE' : ''} INDEX IF NOT EXISTS ${this.quoteId(indexName)} ON ${this.quoteId( + table, + )} (${cols})` + } +} + +function formatDefault(value: string | number | null): string { + if (typeof value === 'number') return String(value) + if (value === null) return 'NULL' + return `'${String(value).replace(/'/g, '\'\'')}'` +} + +export function createDialect(name: DialectName): Dialect { + return new SqlDialect(name) +} diff --git a/src/storage/driver/introspect.ts b/src/storage/driver/introspect.ts new file mode 100644 index 0000000..296aaf7 --- /dev/null +++ b/src/storage/driver/introspect.ts @@ -0,0 +1,37 @@ +import { SqlDriver } from './types' + +/** + * 模式自省。两种方言问法不同，但调用方不该关心。 + * + * 抽出来是因为有两处都要用：storage 初始化时补列，迁移工具决定搬哪些列。 + */ + +/** 这张表在不在。 */ +export async function tableExists(driver: SqlDriver, table: string): Promise { + if (driver.dialect.isPostgres) { + const row = await driver.get<{ count: number }>( + 'SELECT COUNT(*) AS "count" FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = ?', + [table], + ) + return Number(row?.count || 0) > 0 + } + const row = await driver.get<{ name: string }>( + 'SELECT name FROM sqlite_master WHERE type = \'table\' AND name = ?', + [table], + ) + return Boolean(row) +} + +/** 表上现有的列名。表不存在时返回空集合。 */ +export async function listColumns(driver: SqlDriver, table: string): Promise> { + if (driver.dialect.isPostgres) { + const rows = await driver.all<{ column_name: string }>( + 'SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?', + [table], + ) + return new Set(rows.map(r => r.column_name)) + } + // PRAGMA 不接受参数占位符，所以这里只能内插；quoteId 会挡掉不合法的标识符 + const rows = await driver.all<{ name: string }>(`PRAGMA table_info(${driver.dialect.quoteId(table)})`, []) + return new Set(rows.map(r => r.name)) +} diff --git a/src/storage/driver/placeholders.ts b/src/storage/driver/placeholders.ts new file mode 100644 index 0000000..fe8c718 --- /dev/null +++ b/src/storage/driver/placeholders.ts @@ -0,0 +1,96 @@ +/** + * 把 `?` 占位符翻译成 Postgres 的 `$1..$n`。 + * + * 这样 storage 层可以一直按 SQLite 的写法写 `?`，共用同一份 SQL 字符串，只在 + * pg driver 的执行入口翻译一次——比让每条 SQL 自己去适配方言干净得多。 + * + * 扫描时会跳过单引号字符串、双引号标识符、行注释和块注释，避免把字面量里的 + * 问号也换掉。 + * + * 注意：Postgres 的 jsonb `?` 操作符会被误判成占位符。这里 JSON 一律按 TEXT + * 存，用不到 jsonb；真要用就写成 jsonb_exists() 绕开。 + */ +export function toDollarPlaceholders(sql: string): string { + let out = '' + let index = 0 + let i = 0 + + while (i < sql.length) { + const char = sql[i] + + // 单引号字符串：'' 表示转义的单引号 + if (char === '\'') { + const start = i + i++ + while (i < sql.length) { + if (sql[i] === '\'') { + if (sql[i + 1] === '\'') { + i += 2 + continue + } + i++ + break + } + i++ + } + out += sql.slice(start, i) + continue + } + + // 双引号标识符："" 表示转义的双引号 + if (char === '"') { + const start = i + i++ + while (i < sql.length) { + if (sql[i] === '"') { + if (sql[i + 1] === '"') { + i += 2 + continue + } + i++ + break + } + i++ + } + out += sql.slice(start, i) + continue + } + + // 行注释 + if (char === '-' && sql[i + 1] === '-') { + const end = sql.indexOf('\n', i) + const stop = end === -1 ? sql.length : end + out += sql.slice(i, stop) + i = stop + continue + } + + // 块注释 + if (char === '/' && sql[i + 1] === '*') { + const end = sql.indexOf('*/', i + 2) + const stop = end === -1 ? sql.length : end + 2 + out += sql.slice(i, stop) + i = stop + continue + } + + if (char === '?') { + out += `$${++index}` + i++ + continue + } + + out += char + i++ + } + + return out +} + +/** 统计一条 SQL 里的占位符个数，用于参数个数校验。 */ +export function countPlaceholders(sql: string): number { + const translated = toDollarPlaceholders(sql) + const match = translated.match(/\$(\d+)/g) + if (!match) return 0 + return Math.max(...match.map(m => Number(m.slice(1)))) +} diff --git a/src/storage/driver/postgres.ts b/src/storage/driver/postgres.ts new file mode 100644 index 0000000..5d33742 --- /dev/null +++ b/src/storage/driver/postgres.ts @@ -0,0 +1,165 @@ +import { createDialect } from './dialect' +import { toDollarPlaceholders } from './placeholders' +import { Dialect, PostgresConnectionOptions, RunResult, SqlDriver, TransactionScope } from './types' +import { log } from '../logger' + +/** pg 的最小接口。pg 是可选 peer 依赖，动态 import。 */ +interface PgQueryResult { + rows: Record[] + rowCount: number | null +} + +interface PgClient { + query(sql: string, params?: unknown[]): Promise + release(): void +} + +interface PgPool { + query(sql: string, params?: unknown[]): Promise + connect(): Promise + end(): Promise + on(event: 'error', listener: (error: Error) => void): void +} + +/** + * Postgres driver。 + * + * 和 SQLite 那边最大的不同：这里不需要写者队列。SQLite 全库只有一个写锁，所以 + * 必须自己把写入串起来；Postgres 有 MVCC 和真正的连接池，串行化反而会白白限制 + * 吞吐，并发控制整个交给 pg.Pool。 + * + * SQL 仍按 `?` 书写，在执行入口统一翻译成 $1..$n，storage 层不用关心方言。 + */ +export class PostgresDriver implements SqlDriver { + readonly dialect: Dialect = createDialect('postgres') + + private readonly options: PostgresConnectionOptions + private readonly pool: PgPool + private readonly slowQueryMs: number + private closed = false + + constructor(options: PostgresConnectionOptions, pool: PgPool) { + this.options = options + this.pool = pool + this.slowQueryMs = options.slowQueryMs ?? 500 + } + + get name(): string { + return `postgres:${this.options.database}` + } + + async ready(): Promise { + const client = await this.pool.connect() + client.release() + } + + private async query(sql: string, params: unknown[]): Promise { + const startedAt = Date.now() + try { + return await this.pool.query(toDollarPlaceholders(sql), params) + } finally { + const duration = Date.now() - startedAt + if (duration >= this.slowQueryMs) { + log().warn(`[pg] slow query ${duration}ms: ${sql.trim().slice(0, 120)}`) + } + } + } + + async get>(sql: string, params: unknown[] = []): Promise { + const result = await this.query(sql, params) + return result.rows[0] as T | undefined + } + + async all>(sql: string, params: unknown[] = []): Promise { + const result = await this.query(sql, params) + return result.rows as T[] + } + + /** + * lastID 只在显式写了 RETURNING id 时才有值。SQLite 的 this.lastID 在 + * Postgres 没有对应物，调用方不该依赖它。 + */ + async run(sql: string, params: unknown[] = []): Promise { + const result = await this.query(sql, params) + return { + changes: result.rowCount ?? 0, + lastID: result.rows?.[0]?.id as number | string | undefined, + } + } + + /** exec 用来跑建表之类的多语句脚本，本身不带参数，所以不翻译占位符。 */ + async exec(sql: string): Promise { + await this.pool.query(sql) + } + + async transaction(work: (scope: TransactionScope) => Promise): Promise { + const client = await this.pool.connect() + const run = async (sql: string, params: unknown[] = []): Promise => { + const result = await client.query(toDollarPlaceholders(sql), params) + return { changes: result.rowCount ?? 0, lastID: result.rows?.[0]?.id as number | string | undefined } + } + const get = async (sql: string, params: unknown[] = []): Promise => + (await client.query(toDollarPlaceholders(sql), params)).rows[0] as R | undefined + const all = async (sql: string, params: unknown[] = []): Promise => + (await client.query(toDollarPlaceholders(sql), params)).rows as R[] + + try { + await client.query('BEGIN') + const result = await work({ get, all, run } as TransactionScope) + await client.query('COMMIT') + return result + } catch (error) { + try { + await client.query('ROLLBACK') + } catch (rollbackError) { + log().warn( + `[pg] rollback failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`, + ) + } + throw error + } finally { + client.release() + } + } + + async close(): Promise { + if (this.closed) return + this.closed = true + await this.pool.end() + } +} + +/** pg 是可选依赖，只有真的选了 postgres 才需要装。 */ +export async function createPostgresDriver(options: PostgresConnectionOptions): Promise { + let pg + try { + pg = (await import('pg')).default + } catch (error) { + throw new Error( + '使用 postgres 存储需要先安装 pg：pnpm add pg\n' + + `原始错误：${error instanceof Error ? error.message : String(error)}`, + ) + } + const pool = new pg.Pool({ + host: options.host || '127.0.0.1', + port: options.port || 5432, + database: options.database, + user: options.username, + password: options.password, + max: options.pool?.max ?? 10, + idleTimeoutMillis: options.pool?.idle ?? 10000, + connectionTimeoutMillis: options.pool?.acquire ?? 30000, + ssl: options.ssl ?? false, + }) as unknown as PgPool + pool.on('error', error => log().error(`[pg] idle client error: ${error.message}`)) + const driver = new PostgresDriver(options, pool) + try { + await driver.ready() + } catch (error) { + // 首次连接失败时必须把 pool 关掉，否则它的重连计时器会把进程吊住， + // 连接也泄漏了 + await pool.end().catch(() => {}) + throw error + } + return driver +} diff --git a/src/storage/driver/registry.ts b/src/storage/driver/registry.ts new file mode 100644 index 0000000..08230bf --- /dev/null +++ b/src/storage/driver/registry.ts @@ -0,0 +1,134 @@ +import path from 'node:path' +import { SqliteDriver } from './sqlite' +import { createPostgresDriver } from './postgres' +import { ConnectionOptions, LogicalDatabase, SqlDriver } from './types' + +/** + * driver 注册表。 + * + * 逻辑上有三个库：main（渠道、预设、工具等）、history（对话历史）、 + * operation_logs（操作日志）。SQLite 下它们是三个文件——历史库会长到几百 MB， + * 拆开是为了让 VACUUM 只重写真正需要的那个；Postgres 下共用一个连接池，靠表名 + * 区分就够了，再开三个池只是浪费连接数。 + * + * 调用方只认这三个逻辑名，不关心背后是文件还是连接池。 + */ + +const SQLITE_FILES: Record = { + main: 'data.db', + history: 'history.db', + operation_logs: 'operation_logs.db', +} + +export class DriverRegistry { + private readonly drivers = new Map() + private initialized = false + private initPromise: Promise | null = null + private closePromise: Promise | null = null + + /** + * 初始化。并发调用会共用同一个 Promise——否则两个调用方都能通过 + * `initialized === false`，各开一套连接，后完成的那个把前一套从 Map 里顶掉， + * 顶掉的连接就此泄漏。 + * + * 中途失败会把这一轮已经建好的 driver 全部关掉再抛出，不留下半初始化状态； + * initialized 仍为 false，调用方可以修好配置后重试。 + */ + async init(options: ConnectionOptions): Promise { + if (this.initialized) return + if (this.initPromise) return this.initPromise + + this.initPromise = this.doInit(options) + .then(() => { + this.initialized = true + }) + .finally(() => { + this.initPromise = null + }) + + return this.initPromise + } + + private async doInit(options: ConnectionOptions): Promise { + // 先建到临时表里，全部成功才提交到 this.drivers + const opened: SqlDriver[] = [] + const staged = new Map() + + try { + if (options.dialect === 'sqlite') { + for (const name of Object.keys(SQLITE_FILES) as LogicalDatabase[]) { + const driver = new SqliteDriver(path.join(options.dataDir, SQLITE_FILES[name]), { + busyTimeout: options.busyTimeout, + slowQueryMs: options.slowQueryMs, + }) + opened.push(driver) + await driver.ready() + staged.set(name, driver) + } + } else if (options.dialect === 'postgres') { + const shared = await createPostgresDriver(options) + opened.push(shared) + for (const name of Object.keys(SQLITE_FILES) as LogicalDatabase[]) { + staged.set(name, shared) + } + } else { + const dialect = (options as { dialect?: string }).dialect + throw new Error(`不支持的数据库方言：${dialect}，可选值：sqlite、postgres`) + } + } catch (error) { + // 已经打开的别留着——尤其 SQLite 是逐个文件打开的，第二个失败时第一个 + // 还握着文件句柄 + for (const driver of new Set(opened)) { + await driver.close().catch(() => {}) + } + throw error + } + + for (const [name, driver] of staged) { + this.drivers.set(name, driver) + } + } + + get(name: LogicalDatabase): SqlDriver { + const driver = this.drivers.get(name) + if (!driver) { + throw new Error(`driver '${name}' 尚未初始化，请先调用 init()`) + } + return driver + } + + isInitialized(): boolean { + return this.initialized + } + + /** 列出去重后的 driver。Postgres 下三个逻辑库共用一个实例。 */ + listUnique(): SqlDriver[] { + return [...new Set(this.drivers.values())] + } + + /** 关闭全部连接。重复调用共用同一个 Promise。 */ + async close(): Promise { + if (this.closePromise) return this.closePromise + + this.closePromise = (async () => { + for (const driver of this.listUnique()) { + await driver.close() + } + this.drivers.clear() + this.initialized = false + })().finally(() => { + this.closePromise = null + }) + + return this.closePromise + } + + /** 仅供测试：注入一个现成的 driver。 */ + register(name: LogicalDatabase, driver: SqlDriver): void { + this.drivers.set(name, driver) + this.initialized = true + } +} + +/** 进程级默认注册表。宿主插件一般用这个就够了。 */ +export const drivers = new DriverRegistry() diff --git a/src/storage/driver/sqlite.ts b/src/storage/driver/sqlite.ts new file mode 100644 index 0000000..67a8cbb --- /dev/null +++ b/src/storage/driver/sqlite.ts @@ -0,0 +1,376 @@ +import path from 'node:path' +import fs from 'node:fs' +import { createDialect } from './dialect' +import { Dialect, RunResult, SqlDriver, TransactionScope, WriteOptions } from './types' +import { log } from '../logger' + +/** + * sqlite3 的最小接口。sqlite3 是可选 peer 依赖，动态 import，所以这里不引它的 + * 类型，只描述用到的部分。 + */ +interface NativeDatabase { + run(sql: string, params: unknown[], callback: (this: { lastID: number, changes: number }, err: Error | null) => void): void + get(sql: string, params: unknown[], callback: (err: Error | null, row: unknown) => void): void + all(sql: string, params: unknown[], callback: (err: Error | null, rows: unknown[]) => void): void + exec(sql: string, callback: (err: Error | null) => void): void + configure(option: string, value: unknown): void + close(callback: (err: Error | null) => void): void +} + +interface QueueItem { + task: (db: NativeDatabase) => Promise + label: string + queuedAt: number + resolve: (value: T) => void + reject: (error: unknown) => void +} + +type Priority = 'high' | 'normal' | 'low' + +async function openNativeDatabase(dbPath: string, busyTimeout: number): Promise { + let sqlite3 + try { + sqlite3 = (await import('sqlite3')).default + } catch (error) { + throw new Error( + 'SQLite 存储需要安装 sqlite3：pnpm add sqlite3\n' + + `原始错误：${error instanceof Error ? error.message : String(error)}`, + ) + } + return new Promise((resolve, reject) => { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }) + const db = new sqlite3.Database(dbPath, (error: Error | null) => { + if (error) return reject(error) + db.configure('busyTimeout', busyTimeout) + db.exec( + ` + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = ON; + PRAGMA busy_timeout = ${busyTimeout}; + `, + (pragmaError: Error | null) => (pragmaError ? reject(pragmaError) : resolve(db as unknown as NativeDatabase)), + ) + }) + }) +} + +export interface SqliteDriverOptions { + busyTimeout?: number + slowQueryMs?: number +} + +/** + * SQLite driver。 + * + * SQLite 全库只有一个写锁，并发写会直接撞上 SQLITE_BUSY，所以这里自己把写入 + * 串起来：一个专用的 writer 连接 + 三档优先级队列，读取走另一条连接绕开队列。 + * 这套东西 chatgpt-plugin 里已经跑了很久，karin 那份拷贝没有——收进 chaite + * 就是为了让两边都拿到。 + * + * **串行化只在单个实例内生效。** 同一个文件被构造出两个 SqliteDriver 时，两个 + * 队列互不知情，最终还是退回 WAL + busy_timeout 去扛。DriverRegistry 每个文件 + * 只建一个实例，走注册表就没问题；绕过注册表自己 new 的话，这一层保证就没了。 + */ +export class SqliteDriver implements SqlDriver { + readonly dialect: Dialect = createDialect('sqlite') + readonly dbPath: string + + private readonly busyTimeout: number + private readonly slowQueryMs: number + private readonly queues: Record = { high: [], normal: [], low: [] } + private readonly writerReady: Promise + private readonly readerReady: Promise + private readonly readyPromise: Promise + private writerActive = false + private busyCount = 0 + /** + * open -> closing -> closed。 + * + * 只有一个布尔量不够：close() 先置位再等队列排空，中间这段时间里新的 + * run()/get() 如果不被拦下，就会排到一个即将被关掉的连接上，或者留下一个 + * 永远不会完成的 Promise。 + */ + private state: 'open' | 'closing' | 'closed' = 'open' + private closePromise: Promise | null = null + + constructor(dbPath: string, options: SqliteDriverOptions = {}) { + this.dbPath = path.resolve(dbPath) + this.busyTimeout = options.busyTimeout ?? 5000 + this.slowQueryMs = options.slowQueryMs ?? 500 + this.writerReady = openNativeDatabase(this.dbPath, this.busyTimeout) + this.readerReady = this.writerReady.then(() => openNativeDatabase(this.dbPath, this.busyTimeout)) + this.readyPromise = Promise.all([this.writerReady, this.readerReady]) + } + + get name(): string { + return `sqlite:${path.basename(this.dbPath)}` + } + + private get queueDepth(): number { + return this.queues.high.length + this.queues.normal.length + this.queues.low.length + } + + async ready(): Promise { + await this.readyPromise + } + + /** + * 关闭中/已关闭就别再收活了，早失败好过半路连接被抽走。 + * + * 返回 Error 而不是直接抛：driver 的其他方法一律以 rejected promise 报错， + * 这里同步抛会让调用方要写两种错误处理。 + */ + private closedError(operation: string): Error | null { + if (this.state === 'open') return null + return new Error(`[SQLite:${this.name}] driver is ${this.state}, refusing ${operation}`) + } + + private enqueue(task: (db: NativeDatabase) => Promise, options: WriteOptions = {}): Promise { + const refused = this.closedError(options.label || 'write') + if (refused) return Promise.reject(refused) + const { priority = 'normal', label = 'write' } = options + const queuedAt = Date.now() + return new Promise((resolve, reject) => { + const item = { task, label, queuedAt, resolve, reject } as QueueItem + const queue = this.queues[priority] ?? this.queues.normal + queue.push(item as QueueItem) + void this.drain() + }) + } + + private async drain(): Promise { + if (this.writerActive) return + const item = this.queues.high.shift() || this.queues.normal.shift() || this.queues.low.shift() + if (!item) return + + this.writerActive = true + const waitMs = Date.now() - item.queuedAt + if (waitMs >= this.slowQueryMs) { + log().warn(`[SQLite:${this.name}] writer wait ${waitMs}ms (${item.label}), queue=${this.queueDepth}`) + } + const startedAt = Date.now() + try { + const db = await this.writerReady + item.resolve(await item.task(db)) + } catch (error) { + if ((error as { code?: string })?.code === 'SQLITE_BUSY') { + this.busyCount++ + log().warn(`[SQLite:${this.name}] SQLITE_BUSY #${this.busyCount} (${item.label})`) + } + item.reject(error) + } finally { + const duration = Date.now() - startedAt + if (duration >= this.slowQueryMs) { + log().warn(`[SQLite:${this.name}] slow write ${duration}ms (${item.label})`) + } + this.writerActive = false + queueMicrotask(() => void this.drain()) + } + } + + /** 读取直接打在 reader 连接上，绕开写入队列。 */ + private async read(method: 'get' | 'all', sql: string, params: unknown[]): Promise { + const refused = this.closedError(method) + if (refused) throw refused + const db = await this.readerReady + const startedAt = Date.now() + try { + return await new Promise((resolve, reject) => { + const callback = (error: Error | null, result: unknown) => (error ? reject(error) : resolve(result as T)) + if (method === 'get') db.get(sql, params, callback) + else db.all(sql, params, callback as (err: Error | null, rows: unknown[]) => void) + }) + } catch (error) { + if ((error as { code?: string })?.code === 'SQLITE_BUSY') { + this.busyCount++ + log().warn(`[SQLite:${this.name}] SQLITE_BUSY #${this.busyCount} (${method})`) + } + throw error + } finally { + const duration = Date.now() - startedAt + if (duration >= this.slowQueryMs) { + log().warn(`[SQLite:${this.name}] slow ${method} ${duration}ms`) + } + } + } + + get>(sql: string, params: unknown[] = []): Promise { + return this.read('get', sql, params) + } + + all>(sql: string, params: unknown[] = []): Promise { + return this.read('all', sql, params) + } + + run(sql: string, params: unknown[] = [], options: WriteOptions = {}): Promise { + return this.enqueue( + db => + new Promise((resolve, reject) => { + db.run(sql, params, function (error) { + if (error) reject(error) + else resolve({ lastID: this.lastID, changes: this.changes }) + }) + }), + { ...options, label: options.label || sql.trim().split(/\s+/, 2).join(' ') }, + ) + } + + exec(sql: string, options: WriteOptions = {}): Promise { + return this.enqueue( + db => new Promise((resolve, reject) => db.exec(sql, error => (error ? reject(error) : resolve()))), + { label: 'exec', ...options }, + ) + } + + transaction(work: (scope: TransactionScope) => Promise, options: WriteOptions = {}): Promise { + return this.enqueue( + async db => { + const run = (sql: string, params: unknown[] = []) => + new Promise((resolve, reject) => { + db.run(sql, params, function (error) { + if (error) reject(error) + else resolve({ lastID: this.lastID, changes: this.changes }) + }) + }) + const get = (sql: string, params: unknown[] = []) => + new Promise((resolve, reject) => + db.get(sql, params, (error, row) => (error ? reject(error) : resolve(row as R))), + ) + const all = (sql: string, params: unknown[] = []) => + new Promise((resolve, reject) => + db.all(sql, params, (error, rows) => (error ? reject(error) : resolve(rows as R[]))), + ) + + await run('BEGIN IMMEDIATE') + try { + const result = await work({ get, all, run } as TransactionScope) + await run('COMMIT') + return result + } catch (error) { + try { + await run('ROLLBACK') + } catch (rollbackError) { + log().warn( + `[SQLite:${this.name}] rollback failed: ${ + rollbackError instanceof Error ? rollbackError.message : String(rollbackError) + }`, + ) + } + throw error + } + }, + { ...options, label: options.label || 'transaction' }, + ) + } + + /** 库文件加上 WAL/SHM 的总大小。 */ + fileSize(): number { + let total = 0 + for (const suffix of ['', '-wal', '-shm']) { + try { + total += fs.statSync(this.dbPath + suffix).size + } catch { + // 文件不存在就当 0 + } + } + return total + } + + /** + * 回收删除后留下的空闲页。 + * + * 只能用整库 VACUUM：即使把 auto_vacuum 设成 INCREMENTAL（而且必须设在 + * journal_mode=WAL 之前，否则会被静默忽略），PRAGMA incremental_vacuum 在 + * WAL 下也几乎回收不到东西。 + * + * VACUUM 本身很快，但会重写整个数据库文件，期间独占写锁，并且临时需要约等于 + * 库大小的额外磁盘空间。 + */ + async vacuum({ minFreePages = 0 }: { minFreePages?: number } = {}): Promise> { + const before = this.fileSize() + const row = await this.get<{ freelist_count?: number }>('PRAGMA freelist_count', []) + const freePages = row?.freelist_count || 0 + + if (freePages < minFreePages) { + return { name: this.name, path: this.dbPath, skipped: 'free-pages', freePages, before, after: before, ms: 0 } + } + + // VACUUM 要把整个库重写一遍，磁盘剩余空间不够就别开始 + try { + const stat = fs.statfsSync(path.dirname(this.dbPath)) + const available = stat.bavail * stat.bsize + if (available < before * 2) { + return { + name: this.name, path: this.dbPath, skipped: 'disk-space', freePages, before, after: before, ms: 0, available, + } + } + } catch { + // 拿不到磁盘信息就继续，VACUUM 失败会自己报错 + } + + const startedAt = Date.now() + // 读取绕过队列直接打在 reader 上，所以 VACUUM 期间来一次查询就可能拿不到 + // 独占锁而 SQLITE_BUSY。重试几次即可，VACUUM 是幂等的。 + let attempt = 0 + for (;;) { + try { + // low 优先级：排在实时对话的写入后面 + await this.exec('VACUUM', { priority: 'low', label: 'vacuum' }) + break + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const busy = (error as { code?: string })?.code === 'SQLITE_BUSY' || /database is locked/i.test(message) + if (!busy || ++attempt >= 3) throw error + log().warn(`[SQLite:${this.name}] vacuum busy, retry ${attempt}/3`) + await new Promise(resolve => setTimeout(resolve, 2000 * attempt)) + } + } + + // checkpoint 只是顺手把 WAL 截短，失败不该把已经成功的 VACUUM 报成失败 + try { + await this.exec('PRAGMA wal_checkpoint(TRUNCATE)', { priority: 'low', label: 'vacuum checkpoint' }) + } catch (error) { + log().warn( + `[SQLite:${this.name}] post-vacuum checkpoint failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ) + } + + const after = this.fileSize() + const ms = Date.now() - startedAt + log().info( + `[SQLite:${this.name}] vacuum reclaimed ${((before - after) / 1024 / 1024).toFixed(1)} MiB in ${ms}ms`, + ) + return { name: this.name, path: this.dbPath, freePages, before, after, ms } + } + + /** + * 关闭连接。重复调用会等同一个 Promise，而不是第二个调用方立刻拿到 resolve + * 却其实还没关完。 + */ + async close(): Promise { + if (this.closePromise) return this.closePromise + if (this.state === 'closed') return + + this.state = 'closing' + this.closePromise = (async () => { + // 等队列排空，否则正在写的事务会被切断。此时 closedError 已经挡住新任务， + // 所以这个循环一定会收敛。 + while (this.writerActive || this.queueDepth > 0) { + await new Promise(resolve => setTimeout(resolve, 10)) + } + const writer = await this.writerReady + const reader = await this.readerReady + await new Promise((resolve, reject) => reader.close(error => (error ? reject(error) : resolve()))) + await new Promise((resolve, reject) => + writer.run('PRAGMA wal_checkpoint(PASSIVE)', [], error => (error ? reject(error) : resolve())), + ) + await new Promise((resolve, reject) => writer.close(error => (error ? reject(error) : resolve()))) + this.state = 'closed' + })() + + return this.closePromise + } +} diff --git a/src/storage/driver/types.ts b/src/storage/driver/types.ts new file mode 100644 index 0000000..4cf2055 --- /dev/null +++ b/src/storage/driver/types.ts @@ -0,0 +1,107 @@ +/** + * SQL driver 契约。 + * + * chaite 定义了 ChaiteStorage 接口却没给任何具体的 SQL 实现，结果是每个宿主 + * 插件各写一套：chatgpt-plugin 和 karin-plugin-chatgpt 里各有一份逐行雷同的 + * 九个 storage，而且已经开始分叉——只有前者有 WAL、写入队列和 VACUUM。 + * + * 这个模块把那一份实现收到 chaite 里，宿主只需要选方言。 + */ + +export type DialectName = 'sqlite' | 'postgres' + +/** run 的结果。lastID 只有 SQLite 和显式 RETURNING 的场景才有值。 */ +export interface RunResult { + changes: number + lastID?: number | string +} + +/** 事务内可用的操作，和顶层同形。 */ +export interface TransactionScope { + get>(sql: string, params?: unknown[]): Promise + all>(sql: string, params?: unknown[]): Promise + run(sql: string, params?: unknown[]): Promise +} + +/** 写入排队时的提示，只有 SQLite driver 会用到。 */ +export interface WriteOptions { + priority?: 'high' | 'normal' | 'low' + label?: string +} + +export interface Dialect { + readonly name: DialectName + readonly isSqlite: boolean + readonly isPostgres: boolean + columnType(logicalType: ColumnTypeName): string + quoteId(name: string): string + autoIncrementPk(): string + upsert(table: string, columns: string[], conflictColumn?: string, immutable?: string[]): string + createTable(table: string, columns: Record): string + createIndex(table: string, columns: IndexColumn[], options?: IndexOptions): string +} + +/** 索引列。需要给单列指定顺序时用对象形式，例如 (userId, timestamp DESC)。 */ +export type IndexColumn = string | { column: string, order?: 'ASC' | 'DESC' } + +export type ColumnTypeName = 'text' | 'int' | 'bigint' | 'bool' | 'json' | 'blob' + +export interface ColumnSpec { + type?: ColumnTypeName + pk?: boolean + notNull?: boolean + autoIncrement?: boolean + default?: string | number | null +} + +export interface IndexOptions { + name?: string + unique?: boolean + /** 整个索引的默认顺序；单列可以用 IndexColumn 覆盖。 */ + order?: 'ASC' | 'DESC' +} + +/** + * 宿主拿到的就是这个。底下是一个 SQLite 文件还是一个 Postgres 连接池， + * 调用方不需要知道。 + */ +export interface SqlDriver { + readonly dialect: Dialect + readonly name: string + ready(): Promise + get>(sql: string, params?: unknown[]): Promise + all>(sql: string, params?: unknown[]): Promise + run(sql: string, params?: unknown[], options?: WriteOptions): Promise + exec(sql: string, options?: WriteOptions): Promise + transaction(work: (scope: TransactionScope) => Promise, options?: WriteOptions): Promise + close(): Promise +} + +/** 逻辑库名。SQLite 下对应三个文件，Postgres 下共用一个连接池。 */ +export type LogicalDatabase = 'main' | 'history' | 'operation_logs' + +export interface SqliteConnectionOptions { + dialect: 'sqlite' + /** SQLite 文件所在目录 */ + dataDir: string + busyTimeout?: number + slowQueryMs?: number +} + +export interface PostgresConnectionOptions { + dialect: 'postgres' + host?: string + port?: number + database: string + username?: string + password?: string + ssl?: boolean | Record + slowQueryMs?: number + pool?: { + max?: number + idle?: number + acquire?: number + } +} + +export type ConnectionOptions = SqliteConnectionOptions | PostgresConnectionOptions diff --git a/src/storage/index.ts b/src/storage/index.ts new file mode 100644 index 0000000..be91855 --- /dev/null +++ b/src/storage/index.ts @@ -0,0 +1,41 @@ +/** + * SQL 存储层。 + * + * chaite 一直只定义 ChaiteStorage 接口而不给具体实现，结果是每个宿主插件各写一 + * 套：chatgpt-plugin 和 karin-plugin-chatgpt 里各有一份逐行雷同的九个 storage， + * 并且已经分叉——只有前者有 WAL、写入队列和 VACUUM。 + * + * 这个模块把那份实现收进来，宿主只需要选方言： + * + * ```ts + * import { drivers, SqlChannelStorage } from 'chaite' + * + * await drivers.init({ dialect: 'sqlite', dataDir: './data' }) + * const channels = new SqlChannelStorage(drivers.get('main')) + * await channels.initialize() + * ``` + * + * sqlite3 与 pg 都是可选 peer 依赖，按选定的方言二选一安装即可。 + */ + +export * from './driver/types' +export * from './driver/dialect' +export * from './driver/placeholders' +export * from './driver/sqlite' +export * from './driver/postgres' +export * from './driver/registry' + +export * from './sql_storage' + +export * from './sql/shareable' +export * from './sql/base64_image' +export * from './sql/channel_storage' +export * from './sql/chat_preset_storage' +export * from './sql/executable_storage' +export * from './sql/processors_storage' +export * from './sql/tool_groups_storage' +export * from './sql/user_state_storage' +export * from './sql/mcp_server_storage' +export * from './sql/operation_log_storage' +export * from './sql/history_manager' +export * from './migrate' diff --git a/src/storage/logger.ts b/src/storage/logger.ts new file mode 100644 index 0000000..59e661f --- /dev/null +++ b/src/storage/logger.ts @@ -0,0 +1,33 @@ +import type { ILogger } from '../types/common' + +/** + * 存储层的日志入口。 + * + * 这里刻意只用 `import type` 引 ILogger——类型在编译后会被抹掉，于是存储层不会 + * 在模块顶层静态依赖 types 桶文件。那个桶文件会顺带拉进 trigger（依赖只发 ESM + * 的 node-schedule）和 rag/PdfParser（用了 import.meta），一个只想读写 SQLite + * 的模块没有理由把这些都拖上。 + * + * 真正的 getLogger 由包入口在加载时注入（见 src/index.ts）。没注入时退回 console， + * 这样直接 import 深层路径的单元测试也能跑。 + */ +type LoggerResolver = () => ILogger + +let resolver: LoggerResolver | undefined + +export function setStorageLoggerResolver(fn: LoggerResolver): void { + resolver = fn +} + +const consoleLogger: ILogger = { + debug: (...args: unknown[]) => console.debug(...args), + info: (...args: unknown[]) => console.info(...args), + warn: (...args: unknown[]) => console.warn(...args), + error: (...args: unknown[]) => console.error(...args), + trace: (...args: unknown[]) => console.debug(...args), + mark: (...args: unknown[]) => console.info(...args), +} as unknown as ILogger + +export function log(): ILogger { + return resolver?.() ?? consoleLogger +} diff --git a/src/storage/migrate.pg.test.ts b/src/storage/migrate.pg.test.ts new file mode 100644 index 0000000..cc214bd --- /dev/null +++ b/src/storage/migrate.pg.test.ts @@ -0,0 +1,169 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { PGlite } from '@electric-sql/pglite' +import { PostgresDriver } from './driver/postgres' +import { SqliteDriver } from './driver/sqlite' +import { SqlHistoryManager } from './sql/history_manager' +import { SqlOperationLogStorage } from './sql/operation_log_storage' +import { SqlUserStateStorage } from './sql/user_state_storage' +import { migrateSqlStorage, getChaiteTables, MigratableTable } from './migrate' +import { LogicalDatabase, SqlDriver } from './driver/types' +import { HistoryMessage } from '../types/models' +import { OperationLog } from '../types/operation-log' +import { UserState } from '../types/storage' + +/** + * SQLite -> Postgres 的整体搬迁。 + * + * 换方言不会自动带走数据，所以「完全迁到 Postgres」必须有这一步，否则等于从空库 + * 重新开始。这里在真 Postgres（PGlite）上跑完整流程：建表、搬数据、验数、再跑一遍 + * 确认幂等。 + * + * 源端的数据用原始 SQL 灌进去，不经过 DTO：一来避开 chaite 现有的模块环（DTO 那 + * 条链在 jest 的 CJS 下加载不了），二来这样测的就是「行进、行出」，正是迁移工具 + * 该负责的范围。 + */ + +function poolFromPGlite(db: PGlite) { + const query = async (sql: string, params?: unknown[]) => { + const result = await db.query(sql, params as never[]) + return { + rows: (result.rows || []) as Record[], + rowCount: result.affectedRows ?? (result.rows?.length ?? 0), + } + } + return { query, connect: async () => ({ query, release: () => {} }), end: async () => db.close(), on: () => {} } +} + +async function makePg(): Promise { + const db = new PGlite() + await db.waitReady + return new PostgresDriver({ dialect: 'postgres', database: 'pglite' }, poolFromPGlite(db) as never) +} + +function tempDir(prefix: string) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) +} + +/** 用迁移工具自己的表定义把源表建出来，保证两边形状一致。 */ +async function createTable(driver: SqlDriver, definition: MigratableTable) { + await driver.exec(driver.dialect.createTable(definition.table, definition.columns)) +} + +describe('SQLite -> Postgres migration', () => { + test('moves every chaite table and is safe to re-run', async () => { + const dataDir = tempDir('mig-src-') + const imagesDir = tempDir('mig-img-') + const tables = getChaiteTables() + const find = (name: string) => tables.find(t => t.table === name)! + + const sqliteMain = new SqliteDriver(path.join(dataDir, 'data.db')) + const sqliteHistory = new SqliteDriver(path.join(dataDir, 'history.db')) + const sqliteLogs = new SqliteDriver(path.join(dataDir, 'operation_logs.db')) + await Promise.all([sqliteMain.ready(), sqliteHistory.ready(), sqliteLogs.ready()]) + + // channels：直接写行，模拟一个已经在用的库 + await createTable(sqliteMain, find('channels')) + for (let i = 0; i < 5; i++) { + await sqliteMain.run( + 'INSERT INTO "channels" ("id","name","adapterType","type","weight","models","embedded") VALUES (?,?,?,?,?,?,?)', + [`c${i}`, `渠道${i}`, 'openai', 'openai', i + 1, JSON.stringify([{ name: 'gpt-5', features: ['chat'] }]), 0] + ) + } + + await createTable(sqliteMain, find('tools_groups')) + await sqliteMain.run( + 'INSERT INTO "tools_groups" ("id","name","toolIds","isDefault") VALUES (?,?,?,?)', + ['g1', '组', JSON.stringify(['t1', 't2']), 1] + ) + + const userStates = new SqlUserStateStorage(sqliteMain) + await userStates.initialize() + await userStates.setItem('10001', { + userId: '10001', nickname: '昵称', conversations: [], settings: {}, current: {}, + } as unknown as UserState) + + const history = new SqlHistoryManager(sqliteHistory, imagesDir) + await history.initialize() + const messages: HistoryMessage[] = [] + for (let i = 0; i < 250; i++) { + messages.push({ + id: `m${String(i).padStart(4, '0')}`, + parentId: i ? `m${String(i - 1).padStart(4, '0')}` : null, + role: 'user', + content: [{ type: 'text', text: `消息 ${i}` }], + } as unknown as HistoryMessage) + } + await history.saveHistories(messages, 'conv1') + + const logs = new SqlOperationLogStorage(sqliteLogs, 0) + await logs.initialize() + for (let i = 0; i < 20; i++) { + await logs.setItem(`l${i}`, { id: `l${i}`, timestamp: i, type: 'llm.call', level: 'info', summary: `l${i}` } as OperationLog) + } + + // 目标端：一个空的 Postgres，三个逻辑库共用 + const pg = await makePg() + const target = (_db: LogicalDatabase) => pg + const source = (db: LogicalDatabase) => + db === 'history' ? sqliteHistory : db === 'operation_logs' ? sqliteLogs : sqliteMain + + // 小 batchSize，逼出分页游标的逻辑 + const result = await migrateSqlStorage({ from: source, to: target, batchSize: 64 }) + + const byTable = Object.fromEntries(result.tables.map(t => [t.table, t])) + expect(byTable.channels.copied).toBe(5) + expect(byTable.tools_groups.copied).toBe(1) + expect(byTable.user_states.copied).toBe(1) + expect(byTable.history.copied).toBe(250) + expect(byTable.operation_logs.copied).toBe(20) + // 从没写过的表在源端不存在，应当跳过而不是报错 + expect(byTable.chat_presets.skipped).toBe(true) + expect(byTable.tools.skipped).toBe(true) + + // 目标端的数据要和源端一致 + const pgChannels = await pg.all<{ id: string, name: string, weight: number }>( + 'SELECT * FROM "channels" ORDER BY "id"', [] + ) + expect(pgChannels).toHaveLength(5) + expect(pgChannels[3].name).toBe('渠道3') + expect(Number(pgChannels[3].weight)).toBe(4) + + const pgGroup = await pg.get<{ toolIds: string, isDefault: number }>( + 'SELECT * FROM "tools_groups" WHERE "id" = ?', ['g1'] + ) + expect(JSON.parse(pgGroup!.toolIds)).toEqual(['t1', 't2']) + + const pgUser = new SqlUserStateStorage(pg) + await pgUser.initialize() + expect((await pgUser.getItem('10001'))?.nickname).toBe('昵称') + + // 整条 250 层的链要能在 Postgres 上一次查完 + const pgHistory = new SqlHistoryManager(pg, imagesDir) + await pgHistory.initialize() + const chain = await pgHistory.getHistory('m0249') + expect(chain).toHaveLength(250) + expect(chain[0].id).toBe('m0000') + expect(chain[249].id).toBe('m0249') + + const pgLogs = new SqlOperationLogStorage(pg, 0) + await pgLogs.initialize() + expect(await pgLogs.listItems()).toHaveLength(20) + + // 再跑一遍：走 upsert，不该产生重复行 + const second = await migrateSqlStorage({ from: source, to: target, batchSize: 64 }) + expect(second.total).toBe(result.total) + expect(await pg.all('SELECT * FROM "channels"', [])).toHaveLength(5) + expect(await pgHistory.getHistory(undefined, 'conv1')).toHaveLength(250) + + await Promise.all([sqliteMain.close(), sqliteHistory.close(), sqliteLogs.close(), pg.close()]) + }, 120000) + + test('the default table list covers every table the storages create', () => { + expect(getChaiteTables().map(t => t.table).sort()).toEqual([ + 'channels', 'chat_presets', 'history', 'mcp_servers', 'operation_logs', + 'processors', 'tools', 'tools_groups', 'triggers', 'user_states', + ]) + }) +}) diff --git a/src/storage/migrate.ts b/src/storage/migrate.ts new file mode 100644 index 0000000..a1e94c5 --- /dev/null +++ b/src/storage/migrate.ts @@ -0,0 +1,151 @@ +import { SqlDriver } from './driver/types' +import { log } from './logger' +import { CHAITE_TABLES, TableShape } from './sql/tables' +import { listColumns, tableExists } from './driver/introspect' +import { ColumnSpec, ColumnTypeName, LogicalDatabase } from './driver/types' + +/** + * 跨引擎搬数据。 + * + * 换方言不会自动带走数据——sqlite 和 postgres 是两个各自独立的库。这个工具把 + * 一边的表整体复制到另一边，用于「从 SQLite 迁到 Postgres」这种一次性动作。 + * + * 复制走 upsert，所以中断之后重跑是安全的，不会产生重复行。 + * + * **迁移期间必须停止写入。** 这里是逐表、按游标分页复制的，没有全局快照： + * 如果源库还在接受写入，一条已经被游标越过的记录再被更新，这次更新不会进入 + * 目标库。宿主应当先停机再迁移；需要在线迁移的话得另外做增量追赶或双写， + * 这个工具不负责。 + */ + +/** 要搬的表：逻辑库、表名、列定义、主键列。 */ +export interface MigratableTable { + database: LogicalDatabase + table: string + columns: Record + keyColumn: string + /** keyColumn 不是主键时，upsert 依赖的唯一索引 */ + uniqueIndex?: TableShape['uniqueIndex'] + /** 分页读取的排序列，必须唯一。默认用 keyColumn。 */ + cursorColumn?: string +} + +/** + * 默认要搬的表。定义在 sql/tables.ts 里，storage 和这里共用同一份。 + */ +export function getChaiteTables(): MigratableTable[] { + return CHAITE_TABLES.map((shape: TableShape) => ({ + database: shape.database, + table: shape.table, + columns: shape.columns, + keyColumn: shape.keyColumn, + uniqueIndex: shape.uniqueIndex, + })) +} + +export interface MigrateOptions { + /** 源端，按逻辑库取 driver */ + from: (database: LogicalDatabase) => SqlDriver + /** 目标端 */ + to: (database: LogicalDatabase) => SqlDriver + /** 默认搬 getChaiteTables() */ + tables?: MigratableTable[] + /** 每批行数。历史表可能有几十万行，别一次读进内存。 */ + batchSize?: number + /** 每张表搬完回调一次，便于打进度 */ + onProgress?: (progress: TableProgress) => void +} + +export interface TableProgress { + table: string + copied: number + skipped: boolean + reason?: string +} + +export interface MigrateResult { + tables: TableProgress[] + total: number +} + +/** + * 把一批表从源端复制到目标端。 + * + * 目标表不存在会先按列定义建出来，所以可以直接对着一个空库跑。 + */ +export async function migrateSqlStorage(options: MigrateOptions): Promise { + const tables = options.tables || getChaiteTables() + const batchSize = options.batchSize ?? 1000 + const results: TableProgress[] = [] + + for (const definition of tables) { + const source = options.from(definition.database) + const target = options.to(definition.database) + const progress = await migrateTable(source, target, definition, batchSize) + results.push(progress) + options.onProgress?.(progress) + } + + return { tables: results, total: results.reduce((sum, r) => sum + r.copied, 0) } +} + +async function migrateTable( + source: SqlDriver, + target: SqlDriver, + definition: MigratableTable, + batchSize: number, +): Promise { + const { table, columns, keyColumn } = definition + + if (!(await tableExists(source, table))) { + return { table, copied: 0, skipped: true, reason: '源端没有这张表' } + } + + // 目标端建表（已存在就是空操作） + await target.exec(target.dialect.createTable(table, columns)) + // 以及 upsert 依赖的唯一索引，否则 ON CONFLICT 无从匹配 + if (definition.uniqueIndex) { + await target.exec(target.dialect.createIndex(table, definition.uniqueIndex.columns, definition.uniqueIndex)) + } + + const columnNames = Object.keys(columns) + const sourceQ = (name: string) => source.dialect.quoteId(name) + const cursorColumn = definition.cursorColumn || keyColumn + + // 源库可能是更早的版本，少几列；只搬两边都有的列 + const available = await listColumns(source, table) + const copyColumns = columnNames.filter(name => available.has(name)) + if (copyColumns.length === 0) { + return { table, copied: 0, skipped: true, reason: '源端与目标端没有共同列' } + } + + const selectList = copyColumns.map(sourceQ).join(', ') + const upsert = target.dialect.upsert(table, copyColumns, keyColumn) + + let copied = 0 + let cursor: unknown = null + + for (;;) { + // 按游标分页，避免 OFFSET 在大表上越翻越慢 + const where = cursor === null ? '' : ` WHERE ${sourceQ(cursorColumn)} > ?` + const params = cursor === null ? [] : [cursor] + const rows = await source.all>( + `SELECT ${selectList} FROM ${sourceQ(table)}${where} ORDER BY ${sourceQ(cursorColumn)} ASC LIMIT ?`, + [...params, batchSize], + ) + if (rows.length === 0) break + + await target.transaction(async transaction => { + for (const row of rows) { + await transaction.run(upsert, copyColumns.map(name => row[name] ?? null)) + } + }, { priority: 'low', label: `migrate ${table}` }) + + copied += rows.length + cursor = rows[rows.length - 1][cursorColumn] + if (rows.length < batchSize) break + } + + log().info(`[Migrate] ${table}: 复制了 ${copied} 行`) + return { table, copied, skipped: false } +} diff --git a/src/storage/postgres.integration.test.ts b/src/storage/postgres.integration.test.ts new file mode 100644 index 0000000..62feb5f --- /dev/null +++ b/src/storage/postgres.integration.test.ts @@ -0,0 +1,441 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createPostgresDriver, PostgresDriver } from './driver/postgres' +import { SqliteDriver } from './driver/sqlite' +import { DriverRegistry } from './driver/registry' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from './sql_storage' +import { SqlUserStateStorage } from './sql/user_state_storage' +import { SqlOperationLogStorage } from './sql/operation_log_storage' +import { SqlHistoryManager } from './sql/history_manager' +import { migrateSqlStorage, getChaiteTables } from './migrate' +import { LogicalDatabase, PostgresConnectionOptions } from './driver/types' +import { HistoryMessage } from '../types/models' +import { OperationLog } from '../types/operation-log' +import { UserState } from '../types/storage' + +/** + * 对着真实 Postgres 服务端跑的集成测试。 + * + * PGlite 那套验证的是 SQL 语义，但它绕过了 pg.Pool：TCP、认证、acquire/idle + * 超时、连接上限、多 client 并发事务、pool.end() 与在途查询、首次连接失败时的 + * 资源清理——这些只有真服务端能测。 + * + * 没配置连接信息时整组跳过，所以本地和 CI 都不会因为缺少数据库而失败： + * + * CHAITE_PG_HOST=127.0.0.1 CHAITE_PG_PORT=5432 CHAITE_PG_DB=chaite_test \ + * CHAITE_PG_USER=chaite CHAITE_PG_PASSWORD=... pnpm test:pg:integration + */ + +const PG_ENV = { + host: process.env.CHAITE_PG_HOST, + port: process.env.CHAITE_PG_PORT ? Number(process.env.CHAITE_PG_PORT) : undefined, + database: process.env.CHAITE_PG_DB, + username: process.env.CHAITE_PG_USER, + password: process.env.CHAITE_PG_PASSWORD, +} + +const CONFIGURED = Boolean(PG_ENV.host && PG_ENV.database && PG_ENV.username) +const describeIfPg = CONFIGURED ? describe : describe.skip + +function pgOptions(overrides: Partial = {}): PostgresConnectionOptions { + return { + dialect: 'postgres', + host: PG_ENV.host, + port: PG_ENV.port, + database: PG_ENV.database as string, + username: PG_ENV.username, + password: PG_ENV.password, + ...overrides, + } +} + +/** 每个用例都从干净的 schema 开始，互不干扰。 */ +async function resetSchema(): Promise { + const driver = await createPostgresDriver(pgOptions()) + await driver.exec('DROP SCHEMA public CASCADE; CREATE SCHEMA public;') + await driver.close() +} + +interface Widget { id?: string, name?: string, kind?: string, weight?: number } + +const widgetSpec: TableSpec = { + table: 'widgets', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + kind: { type: 'text' }, + weight: { type: 'int', default: 1 }, + }, + indexes: [{ columns: ['kind'] }], + filterable: ['id', 'name', 'kind', 'weight'], + numeric: ['weight'], + toRecord: (entity, id) => ({ id, name: entity.name, kind: entity.kind, weight: entity.weight ?? 1 }), + fromRecord: record => ({ + id: record.id as string, + name: record.name as string, + kind: record.kind as string, + weight: Number(record.weight), + }), +} + +describeIfPg('PostgresDriver against a real server', () => { + beforeEach(resetSchema) + + test('connects over TCP with real authentication', async () => { + const driver = await createPostgresDriver(pgOptions()) + const row = await driver.get<{ db: string, v: string }>('SELECT current_database() AS db, version() AS v', []) + expect(row?.db).toBe(PG_ENV.database) + expect(row?.v).toMatch(/PostgreSQL/) + await driver.close() + }) + + // 服务端用 trust 认证时（CI 就是）任何密码都会被接受，这个用例没有意义。 + // 「首次连接失败不泄漏 pool」这条由下面 unreachable host / 不存在的库覆盖。 + const testIfPassword = PG_ENV.password ? test : test.skip + testIfPassword('a bad password fails fast and does not leak the pool', async () => { + // 关键在于「不挂住」：pool 没 end 的话它的重连计时器会把 jest 吊到超时 + await expect( + createPostgresDriver(pgOptions({ password: 'definitely-not-the-password' })) + ).rejects.toThrow() + }, 30000) + + test('an unreachable host fails fast and does not leak the pool', async () => { + await expect( + createPostgresDriver(pgOptions({ host: '127.0.0.1', port: 1, pool: { acquire: 2000 } })) + ).rejects.toThrow() + }, 30000) + + test('a nonexistent database is reported rather than hanging', async () => { + await expect( + createPostgresDriver(pgOptions({ database: 'no_such_database_here' })) + ).rejects.toThrow() + }, 30000) + + test('concurrent transactions on separate clients stay isolated', async () => { + const driver = await createPostgresDriver(pgOptions({ pool: { max: 5 } })) + await driver.exec('CREATE TABLE counters (id TEXT PRIMARY KEY, n INT)') + await driver.run('INSERT INTO counters VALUES (?, ?)', ['a', 0]) + + // 10 个并发事务各自 +1，行级锁应当让它们串起来，最终正好是 10 + await Promise.all( + Array.from({ length: 10 }, () => + driver.transaction(async tx => { + const row = await tx.get<{ n: number }>('SELECT n FROM counters WHERE id = ? FOR UPDATE', ['a']) + await tx.run('UPDATE counters SET n = ? WHERE id = ?', [Number(row?.n) + 1, 'a']) + }) + ) + ) + + const final = await driver.get<{ n: number }>('SELECT n FROM counters WHERE id = ?', ['a']) + expect(Number(final?.n)).toBe(10) + await driver.close() + }, 60000) + + test('a rolled back transaction leaves nothing behind on the server', async () => { + const driver = await createPostgresDriver(pgOptions()) + await driver.exec('CREATE TABLE t (id TEXT PRIMARY KEY)') + + await expect(driver.transaction(async tx => { + await tx.run('INSERT INTO t VALUES (?)', ['x']) + throw new Error('boom') + })).rejects.toThrow('boom') + + expect(await driver.all('SELECT * FROM t', [])).toHaveLength(0) + // 回滚之后 client 必须干净地还回池子，后续查询照常 + expect(await driver.get('SELECT 1 AS ok', [])).toMatchObject({ ok: 1 }) + await driver.close() + }) + + test('more concurrent work than pool.max queues instead of failing', async () => { + const driver = await createPostgresDriver(pgOptions({ pool: { max: 2 } })) + await driver.exec('CREATE TABLE t (id INT)') + + // 20 个并发查询、池子只有 2 条连接：应该排队而不是报 acquire 失败 + const results = await Promise.all( + Array.from({ length: 20 }, (_, i) => driver.get<{ n: number }>('SELECT ? ::int AS n', [i])) + ) + expect(results.map(r => Number(r?.n)).sort((a, b) => a - b)[19]).toBe(19) + await driver.close() + }, 60000) + + test('acquire timeout surfaces as an error, not a hang', async () => { + const driver = await createPostgresDriver(pgOptions({ pool: { max: 1, acquire: 500 } })) + await driver.exec('CREATE TABLE t (id INT)') + + // 占住唯一那条连接，另一边应当在 acquire 超时后报错 + let release: () => void = () => {} + const held = new Promise(resolve => { release = resolve }) + const holding = driver.transaction(async () => { await held }) + await new Promise(resolve => setTimeout(resolve, 100)) + + await expect(driver.get('SELECT 1', [])).rejects.toThrow(/timeout|timed out/i) + + release() + await holding + await driver.close() + }, 60000) + + test('close() ends the pool and the server sees the connections go', async () => { + const driver = await createPostgresDriver(pgOptions({ pool: { max: 3 } })) + await driver.exec('CREATE TABLE t (id INT)') + await Promise.all([driver.get('SELECT 1', []), driver.get('SELECT 2', []), driver.get('SELECT 3', [])]) + + await driver.close() + // end() 之后再用同一个 driver 必须失败，而不是悄悄重连 + await expect(driver.get('SELECT 1', [])).rejects.toThrow() + + // 从一条新连接确认服务端这边的连接已经收掉 + const probe = await createPostgresDriver(pgOptions()) + const row = await probe.get<{ n: number }>( + 'SELECT count(*) AS n FROM pg_stat_activity WHERE datname = ? AND pid <> pg_backend_pid()', + [PG_ENV.database] + ) + expect(Number(row?.n)).toBe(0) + await probe.close() + }, 60000) + + test('an older table missing columns is reconciled on a real server', async () => { + const driver = await createPostgresDriver(pgOptions()) + // 老版本表：只有主键和 name + await driver.exec('CREATE TABLE widgets (id TEXT PRIMARY KEY, name TEXT NOT NULL)') + await driver.run('INSERT INTO widgets (id, name) VALUES (?, ?)', ['old', 'legacy row']) + + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + const columns = await driver.all<{ column_name: string }>( + 'SELECT column_name FROM information_schema.columns WHERE table_name = ?', ['widgets'] + ) + expect(columns.map(c => c.column_name).sort()).toEqual(['id', 'kind', 'name', 'weight']) + + // 补列之前这里会报 column "kind" of relation "widgets" does not exist + await storage.setItem('new', { name: 'fresh', kind: 'a', weight: 3 }) + expect(await storage.listItems()).toHaveLength(2) + // 老行拿到 weight 的默认值 + expect((await storage.getItem('old'))?.weight).toBe(1) + await driver.close() + }, 60000) + + test('storages round-trip through the real driver', async () => { + const driver = await createPostgresDriver(pgOptions()) + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + await storage.setItem('w1', { name: 'first', kind: 'a', weight: 5 }) + await storage.setItem('w1', { name: 'renamed', kind: 'b', weight: 6 }) + expect(await storage.listItems()).toHaveLength(1) + expect((await storage.getItem('w1'))?.name).toBe('renamed') + expect((await storage.listItemsByEqFilter({ weight: '6' })).map(w => w.id)).toEqual(['w1']) + await driver.close() + }) + + test('user_states dedupe and unique index hold on a real server', async () => { + const driver = await createPostgresDriver(pgOptions()) + await driver.exec(`CREATE TABLE user_states ( + id TEXT PRIMARY KEY, "userId" TEXT NOT NULL, nickname TEXT, card TEXT, + conversations TEXT NOT NULL, settings TEXT NOT NULL, current TEXT NOT NULL, "updatedAt" BIGINT + )`) + const insert = `INSERT INTO user_states (id, "userId", nickname, card, conversations, settings, current, "updatedAt") + VALUES (?, ?, ?, NULL, '[]', '{}', '{}', ?)` + await driver.run(insert, ['uuid-old', '555', 'stale', 1000]) + await driver.run(insert, ['uuid-new', '555', 'fresh', 2000]) + + const storage = new SqlUserStateStorage(driver) + await storage.initialize() + + const rows = await driver.all<{ id: string, nickname: string }>('SELECT * FROM user_states', []) + expect(rows).toHaveLength(1) + expect(rows[0].nickname).toBe('fresh') + + // 唯一索引真的在服务端生效 + await expect(driver.run(insert, ['uuid-third', '555', 'dupe', 3000])).rejects.toThrow() + await driver.close() + }, 60000) + + test('concurrent initialize() on the same table does not fight', async () => { + const driver = await createPostgresDriver(pgOptions({ pool: { max: 5 } })) + // 三个实例同时建同一张表 + 唯一索引，IF NOT EXISTS 要能顶住并发 + const storages = [ + new SqlUserStateStorage(driver), + new SqlUserStateStorage(driver), + new SqlUserStateStorage(driver), + ] + await Promise.all(storages.map(s => s.initialize())) + + const indexes = await driver.all<{ indexname: string }>( + 'SELECT indexname FROM pg_indexes WHERE tablename = ?', ['user_states'] + ) + expect(indexes.map(i => i.indexname)).toContain('uniq_user_states_userId') + await driver.close() + }, 60000) + + test('history chain CTE and pruning work on a real server', async () => { + const driver = await createPostgresDriver(pgOptions()) + const manager = new SqlHistoryManager(driver, fs.mkdtempSync(path.join(os.tmpdir(), 'pgi-img-'))) + await manager.initialize() + + const messages: HistoryMessage[] = Array.from({ length: 400 }, (_, i) => ({ + id: `m${String(i).padStart(4, '0')}`, + parentId: i ? `m${String(i - 1).padStart(4, '0')}` : null, + role: 'user', + content: [{ type: 'text', text: `消息 ${i}` }], + }) as unknown as HistoryMessage) + await manager.saveHistories(messages, 'conv1') + + // 400 层的链必须是一次查询，不是 400 次往返 + const startedAt = Date.now() + const chain = await manager.getHistory('m0399') + const elapsed = Date.now() - startedAt + expect(chain).toHaveLength(400) + expect(chain[0].id).toBe('m0000') + // 真网络上 400 次往返绝不可能在这个时间内跑完，这就是 CTE 生效的证据 + expect(elapsed).toBeLessThan(2000) + + await driver.run('UPDATE history SET "createdAt" = ? WHERE id = ?', ['2000-01-01T00:00:00.000Z', 'm0000']) + expect(await manager.countHistoryBefore({ before: '2001-01-01T00:00:00.000Z' })).toBe(1) + expect((await manager.pruneHistory({ before: '2001-01-01T00:00:00.000Z' })).deleted).toBe(1) + await driver.close() + }, 120000) + + test('operation log retention prunes on a real server', async () => { + const driver = await createPostgresDriver(pgOptions()) + const logs = new SqlOperationLogStorage(driver, 10) + await logs.initialize() + for (let i = 0; i < 40; i++) { + const id = `log-${String(i).padStart(3, '0')}` + await logs.setItem(id, { id, timestamp: i, type: 'llm.call', level: 'info', summary: id } as OperationLog) + } + await logs.setMaxEntries(10) + expect(await driver.all('SELECT id FROM operation_logs', [])).toHaveLength(10) + await driver.close() + }, 60000) +}) + +describeIfPg('DriverRegistry against a real server', () => { + beforeEach(resetSchema) + + test('postgres init shares one pool across all three logical databases', async () => { + const registry = new DriverRegistry() + await registry.init(pgOptions()) + + expect(registry.listUnique()).toHaveLength(1) + expect(registry.get('main')).toBe(registry.get('history')) + expect(registry.get('operation_logs')).toBe(registry.get('main')) + await registry.close() + }) + + test('concurrent init() against a real server opens one pool', async () => { + const registry = new DriverRegistry() + await Promise.all([registry.init(pgOptions()), registry.init(pgOptions()), registry.init(pgOptions())]) + expect(registry.listUnique()).toHaveLength(1) + await registry.close() + + // 并发初始化不该漏下额外连接 + const probe = await createPostgresDriver(pgOptions()) + const row = await probe.get<{ n: number }>( + 'SELECT count(*) AS n FROM pg_stat_activity WHERE datname = ? AND pid <> pg_backend_pid()', + [PG_ENV.database] + ) + expect(Number(row?.n)).toBe(0) + await probe.close() + }, 60000) + + test('a failed postgres init leaves nothing open and stays retryable', async () => { + const registry = new DriverRegistry() + // 用不存在的库来制造失败，而不是用错密码：服务端用 trust 认证时（CI 就是） + // 任何密码都会被接受，那样这个用例就不会失败了 + await expect(registry.init(pgOptions({ database: 'no_such_database_here' }))).rejects.toThrow() + expect(registry.isInitialized()).toBe(false) + expect(registry.listUnique()).toHaveLength(0) + + // 配置修好后还能正常初始化 + await registry.init(pgOptions()) + expect(registry.isInitialized()).toBe(true) + await registry.close() + }, 60000) +}) + +describeIfPg('SQLite -> real Postgres migration', () => { + beforeEach(resetSchema) + + test('moves a populated SQLite install and is idempotent', async () => { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pgi-src-')) + const imagesDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pgi-img-')) + const tables = getChaiteTables() + const find = (name: string) => tables.find(t => t.table === name)! + + const sqliteMain = new SqliteDriver(path.join(dataDir, 'data.db')) + const sqliteHistory = new SqliteDriver(path.join(dataDir, 'history.db')) + const sqliteLogs = new SqliteDriver(path.join(dataDir, 'operation_logs.db')) + await Promise.all([sqliteMain.ready(), sqliteHistory.ready(), sqliteLogs.ready()]) + + await sqliteMain.exec(sqliteMain.dialect.createTable('channels', find('channels').columns)) + for (let i = 0; i < 20; i++) { + await sqliteMain.run( + 'INSERT INTO "channels" ("id","name","adapterType","type","weight","models","embedded") VALUES (?,?,?,?,?,?,?)', + [`c${i}`, `渠道${i}`, 'openai', 'openai', i + 1, JSON.stringify([{ name: 'gpt-5', features: ['chat'] }]), 0] + ) + } + + const userStates = new SqlUserStateStorage(sqliteMain) + await userStates.initialize() + await userStates.setItem('10001', { + userId: '10001', nickname: '昵称', conversations: [], settings: {}, current: {}, + } as unknown as UserState) + + const history = new SqlHistoryManager(sqliteHistory, imagesDir) + await history.initialize() + await history.saveHistories( + Array.from({ length: 1500 }, (_, i) => ({ + id: `m${String(i).padStart(5, '0')}`, + parentId: i ? `m${String(i - 1).padStart(5, '0')}` : null, + role: 'user', + content: [{ type: 'text', text: `消息 ${i}` }], + }) as unknown as HistoryMessage), + 'conv1' + ) + + const logs = new SqlOperationLogStorage(sqliteLogs, 0) + await logs.initialize() + for (let i = 0; i < 50; i++) { + await logs.setItem(`l${i}`, { id: `l${i}`, timestamp: i, type: 'llm.call', level: 'info', summary: `l${i}` } as OperationLog) + } + + const target = await createPostgresDriver(pgOptions({ pool: { max: 4 } })) + const source = (db: LogicalDatabase) => + db === 'history' ? sqliteHistory : db === 'operation_logs' ? sqliteLogs : sqliteMain + + const result = await migrateSqlStorage({ from: source, to: () => target, batchSize: 250 }) + const byTable = Object.fromEntries(result.tables.map(t => [t.table, t])) + expect(byTable.channels.copied).toBe(20) + expect(byTable.user_states.copied).toBe(1) + expect(byTable.history.copied).toBe(1500) + expect(byTable.operation_logs.copied).toBe(50) + + // 搬完之后在 Postgres 上把整条 1500 层的链走一遍 + const pgHistory = new SqlHistoryManager(target, imagesDir) + await pgHistory.initialize() + const chain = await pgHistory.getHistory('m01499') + expect(chain).toHaveLength(1500) + expect(chain[0].id).toBe('m00000') + + const pgUser = new SqlUserStateStorage(target) + await pgUser.initialize() + expect((await pgUser.getItem('10001'))?.nickname).toBe('昵称') + + // 重跑一遍不该产生重复行 + const second = await migrateSqlStorage({ from: source, to: () => target, batchSize: 250 }) + expect(second.total).toBe(result.total) + expect(await target.all('SELECT id FROM channels', [])).toHaveLength(20) + expect(await target.all('SELECT id FROM history', [])).toHaveLength(1500) + + await Promise.all([sqliteMain.close(), sqliteHistory.close(), sqliteLogs.close(), target.close()]) + }, 300000) +}) + +if (!CONFIGURED) { + describe('Postgres integration', () => { + test.skip('skipped: CHAITE_PG_* not configured', () => {}) + }) +} diff --git a/src/storage/postgres.test.ts b/src/storage/postgres.test.ts new file mode 100644 index 0000000..a9b8b14 --- /dev/null +++ b/src/storage/postgres.test.ts @@ -0,0 +1,253 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { PGlite } from '@electric-sql/pglite' +import { PostgresDriver } from './driver/postgres' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from './sql_storage' +import { SqlUserStateStorage } from './sql/user_state_storage' +import { SqlOperationLogStorage } from './sql/operation_log_storage' +import { SqlHistoryManager } from './sql/history_manager' +import { HistoryMessage } from '../types/models' +import { OperationLog } from '../types/operation-log' +import { UserState } from '../types/storage' + +/** + * 真·Postgres 上的验证。 + * + * PGlite 是编译成 WASM 的 Postgres 本体，不是模拟器——语法、类型、ON CONFLICT、 + * 窗口函数的行为都和真实服务端一致。所以这套测试能证明「生成的 SQL 在 Postgres + * 上确实跑得通」，而不只是「字符串拼对了」。 + * + * 剩下没被覆盖的是连接池、网络故障和并发，那些必须对着真实服务端跑。 + */ + +/** 把 PGlite 包成 PostgresDriver 期望的 pg.Pool 形状。 */ +function poolFromPGlite(db: PGlite) { + const query = async (sql: string, params?: unknown[]) => { + const result = await db.query(sql, params as never[]) + return { rows: (result.rows || []) as Record[], rowCount: result.affectedRows ?? (result.rows?.length ?? 0) } + } + return { + query, + // PGlite 是单连接的，事务直接打在同一个实例上 + connect: async () => ({ query, release: () => {} }), + end: async () => db.close(), + on: () => {}, + } +} + +async function makePgDriver(): Promise { + const db = new PGlite() + await db.waitReady + return new PostgresDriver({ dialect: 'postgres', database: 'pglite' }, poolFromPGlite(db) as never) +} + +interface Widget { + id?: string + name?: string + kind?: string + weight?: number + enabled?: boolean + payload?: unknown +} + +const widgetSpec: TableSpec = { + table: 'widgets', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + kind: { type: 'text' }, + weight: { type: 'int', default: 1 }, + enabled: { type: 'bool', default: 0 }, + payload: { type: 'json' }, + }, + indexes: [{ columns: ['kind'] }], + filterable: ['id', 'name', 'kind', 'weight', 'enabled'], + numeric: ['weight'], + boolean: ['enabled'], + toRecord: (entity, id) => ({ + id, + name: entity.name, + kind: entity.kind, + weight: entity.weight ?? 1, + enabled: entity.enabled ? 1 : 0, + payload: stringifyJson(entity.payload), + }), + fromRecord: record => ({ + id: record.id as string, + name: record.name as string, + kind: record.kind as string, + weight: Number(record.weight), + enabled: Boolean(record.enabled), + payload: parseJson(record.payload, {}), + }), +} + +describe('PostgresDriver against a real Postgres engine', () => { + test('creates tables and indexes from the generated DDL', async () => { + const driver = await makePgDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + const tables = await driver.all<{ tablename: string }>( + 'SELECT tablename FROM pg_tables WHERE schemaname = \'public\'', [] + ) + expect(tables.map(t => t.tablename)).toContain('widgets') + + const indexes = await driver.all<{ indexname: string }>( + 'SELECT indexname FROM pg_indexes WHERE tablename = \'widgets\'', [] + ) + expect(indexes.map(i => i.indexname)).toContain('idx_widgets_kind') + await driver.close() + }) + + test('placeholders survive the ? -> $n translation end to end', async () => { + const driver = await makePgDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + await storage.setItem('w1', { name: 'first', kind: 'a', weight: 5, enabled: true, payload: { deep: [1, 2] } }) + const loaded = await storage.getItem('w1') + expect(loaded).toMatchObject({ name: 'first', weight: 5, enabled: true, payload: { deep: [1, 2] } }) + await driver.close() + }) + + test('upsert really upserts on Postgres', async () => { + const driver = await makePgDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + await storage.setItem('w1', { name: 'first', kind: 'a' }) + await storage.setItem('w1', { name: 'renamed', kind: 'b' }) + + const all = await storage.listItems() + expect(all).toHaveLength(1) + expect(all[0]).toMatchObject({ name: 'renamed', kind: 'b' }) + await driver.close() + }) + + test('filters, IN queries and ordering behave the same as on sqlite', async () => { + const driver = await makePgDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + await storage.setItem('w1', { name: 'a', kind: 'x', weight: 1, enabled: true }) + await storage.setItem('w2', { name: 'b', kind: 'x', weight: 2, enabled: false }) + await storage.setItem('w3', { name: 'c', kind: 'y', weight: 1, enabled: true }) + + expect((await storage.listItemsByEqFilter({ kind: 'x' })).map(w => w.id).sort()).toEqual(['w1', 'w2']) + // 数值列传字符串也要能匹配（Postgres 的类型检查比 SQLite 严得多） + expect((await storage.listItemsByEqFilter({ weight: '1' })).map(w => w.id).sort()).toEqual(['w1', 'w3']) + expect((await storage.listItemsByEqFilter({ enabled: true })).map(w => w.id).sort()).toEqual(['w1', 'w3']) + expect( + (await storage.listItemsByInQuery([{ field: 'kind', values: ['x'] }, { field: 'name', values: ['a', 'b'] }])) + .map(w => w.id).sort() + ).toEqual(['w1', 'w2']) + await driver.close() + }) + + test('transactions commit and roll back', async () => { + const driver = await makePgDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + await driver.transaction(async tx => { + await tx.run('INSERT INTO "widgets" ("id","name") VALUES (?,?)', ['ok', 'committed']) + }) + expect(await storage.getItem('ok')).toMatchObject({ name: 'committed' }) + + await expect( + driver.transaction(async tx => { + await tx.run('INSERT INTO "widgets" ("id","name") VALUES (?,?)', ['bad', 'rolled back']) + throw new Error('boom') + }) + ).rejects.toThrow('boom') + expect(await storage.getItem('bad')).toBeNull() + await driver.close() + }) + + test('user_states dedupe uses a window function Postgres accepts', async () => { + const driver = await makePgDriver() + // 先造出老结构和重复行，再让 initialize 去重并建唯一索引 + await driver.exec(`CREATE TABLE user_states ( + id TEXT PRIMARY KEY, "userId" TEXT NOT NULL, nickname TEXT, card TEXT, + conversations TEXT NOT NULL, settings TEXT NOT NULL, current TEXT NOT NULL, "updatedAt" BIGINT + )`) + const insert = `INSERT INTO user_states (id, "userId", nickname, card, conversations, settings, current, "updatedAt") + VALUES (?, ?, ?, NULL, '[]', '{}', '{}', ?)` + await driver.run(insert, ['uuid-old', '555', 'stale', 1000]) + await driver.run(insert, ['uuid-new', '555', 'fresh', 2000]) + + const storage = new SqlUserStateStorage(driver) + await storage.initialize() + + const rows = await driver.all<{ id: string, nickname: string }>('SELECT * FROM user_states', []) + expect(rows).toHaveLength(1) + expect(rows[0].nickname).toBe('fresh') + + // 唯一索引装上之后重复写入只更新那一行，代理主键不变 + await storage.setItem('555', { userId: '555', nickname: 'updated', conversations: [], settings: {}, current: {} } as unknown as UserState) + const after = await driver.all<{ id: string, nickname: string }>('SELECT * FROM user_states', []) + expect(after).toHaveLength(1) + expect(after[0].nickname).toBe('updated') + expect(after[0].id).toBe('uuid-new') + await driver.close() + }) + + test('operation log retention prunes on Postgres', async () => { + const driver = await makePgDriver() + const storage = new SqlOperationLogStorage(driver, 10) + await storage.initialize() + + for (let i = 0; i < 30; i++) { + const id = `log-${String(i).padStart(3, '0')}` + await storage.setItem(id, { id, timestamp: i, type: 'llm.call', level: 'info', summary: id } as OperationLog) + } + await storage.setMaxEntries(10) + + const rows = await driver.all<{ id: string }>('SELECT id FROM operation_logs ORDER BY timestamp ASC', []) + expect(rows).toHaveLength(10) + expect(rows[0].id).toBe('log-020') + await driver.close() + }) + + test('history chain, reparenting and pruning work on Postgres', async () => { + const driver = await makePgDriver() + const imagesDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pg-img-')) + const manager = new SqlHistoryManager(driver, imagesDir) + await manager.initialize() + + const message = (id: string, parentId: string | null, text: string) => + ({ id, parentId, role: 'user', content: [{ type: 'text', text }] }) as unknown as HistoryMessage + + await manager.saveHistory(message('m1', null, 'first'), 'c1') + await manager.saveHistory(message('m2', 'm1', 'second'), 'c1') + await manager.saveHistory(message('m3', 'm2', 'third'), 'c1') + + expect((await manager.getHistory('m3')).map(m => m.id)).toEqual(['m1', 'm2', 'm3']) + + await manager.removeHistory('m2', 'c1') + expect((await manager.getHistory('m3')).map(m => m.id)).toEqual(['m1', 'm3']) + + await driver.run('UPDATE history SET "createdAt" = ? WHERE id = ?', ['2000-01-01T00:00:00.000Z', 'm1']) + const cutoff = '2001-01-01T00:00:00.000Z' + expect(await manager.countHistoryBefore({ before: cutoff })).toBe(1) + expect((await manager.pruneHistory({ before: cutoff })).deleted).toBe(1) + expect((await manager.getHistory(undefined, 'c1')).map(m => m.id)).toEqual(['m3']) + await driver.close() + }) + + test('batch history writes go through one transaction', async () => { + const driver = await makePgDriver() + const manager = new SqlHistoryManager(driver, fs.mkdtempSync(path.join(os.tmpdir(), 'pg-img-'))) + await manager.initialize() + + await manager.saveHistories([ + { id: 'm1', parentId: null, role: 'user', content: [{ type: 'text', text: 'a' }] } as unknown as HistoryMessage, + { id: 'm2', parentId: 'm1', role: 'user', content: [{ type: 'text', text: 'b' }] } as unknown as HistoryMessage, + ], 'c1') + + expect(await manager.getHistory(undefined, 'c1')).toHaveLength(2) + await driver.close() + }) +}) diff --git a/src/storage/sql/base64_image.ts b/src/storage/sql/base64_image.ts new file mode 100644 index 0000000..f031d31 --- /dev/null +++ b/src/storage/sql/base64_image.ts @@ -0,0 +1,33 @@ +/** + * 判断是不是图片数据，但不用正则。 + * + * V8 的正则引擎在超长字符串上可能把自己的栈撑爆——尤其是非法字符出现在靠近 + * 末尾的位置时，而这里的输入可能是几 MB 的 Base64。 + * + * 行为刻意保持宽松，和改造前一致：若干个 Base64 字母表字符，末尾最多两个补位号。 + */ +export function isBase64Image(value: unknown): boolean { + if (!value || typeof value !== 'string') return false + if (value.startsWith('data:image/')) return true + + let contentEnd = value.length + if (value.charCodeAt(contentEnd - 1) === 61) contentEnd-- // = + if (contentEnd > 0 && value.charCodeAt(contentEnd - 1) === 61) contentEnd-- + if (contentEnd === 0) return false + + for (let index = 0; index < contentEnd; index++) { + const code = value.charCodeAt(index) + const isBase64Character = + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + (code >= 48 && code <= 57) || + code === 43 || + code === 47 + if (!isBase64Character) return false + } + + for (let index = contentEnd; index < value.length; index++) { + if (value.charCodeAt(index) !== 61) return false + } + return true +} diff --git a/src/storage/sql/channel_storage.ts b/src/storage/sql/channel_storage.ts new file mode 100644 index 0000000..1ceade2 --- /dev/null +++ b/src/storage/sql/channel_storage.ts @@ -0,0 +1,105 @@ +import { Channel } from '../../channels/channels' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from '../sql_storage' +import { stampTimestamps } from './shareable' +import { CHANNELS } from './tables' + +const spec: TableSpec = { + table: CHANNELS.table, + columns: CHANNELS.columns, + indexes: [{ columns: ['type'] }, { columns: ['status'] }], + filterable: [ + 'id', 'name', 'description', 'adapterType', 'type', 'status', 'cloudId', 'weight', 'priority', 'embedded', + ], + numeric: ['weight', 'priority'], + boolean: ['embedded'], + toRecord(channel, id) { + const { + id: _ignored, name, description, adapterType, type, weight, priority, + status, disabledReason, models, options, statistics, + uploader, cloudId, createdAt, updatedAt, md5, embedded, ...rest + } = channel as Channel & Record + + return { + id, + name: name || '', + description: description || '', + adapterType: adapterType || type || '', + type: type || '', + weight: weight || 1, + priority: priority || 0, + status: status || 'enabled', + disabledReason: disabledReason || null, + models: Array.isArray(models) ? JSON.stringify(models) : '[]', + options: stringifyJson(options), + statistics: stringifyJson(statistics), + uploader: stringifyJson(uploader), + cloudId: cloudId || null, + createdAt: createdAt || '', + updatedAt: updatedAt || '', + md5: md5 || '', + embedded: embedded ? 1 : 0, + extra: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null, + } + }, + fromRecord(record) { + if (!record) return null + return new Channel({ + id: record.id as string, + name: record.name as string, + description: record.description as string, + adapterType: record.adapterType as Channel['adapterType'], + type: record.type as Channel['type'], + weight: Number(record.weight), + priority: Number(record.priority), + status: record.status as Channel['status'], + disabledReason: record.disabledReason as string, + models: parseJson(record.models, []), + // Channel 的构造函数会把这两个普通对象分别喂给 BaseClientOptions.fromString + // 和 ChannelStatistics，所以这里给的是反序列化后的裸对象，不是成品实例 + options: parseJson(record.options, {}) as Channel['options'], + statistics: parseJson(record.statistics, {}) as Channel['statistics'], + uploader: parseJson(record.uploader, undefined), + // 列一直是 INTEGER（老库就是这么建的），但 Shareable 把 cloudId 声明成 + // string。保持列不变以兼容既有数据，这里只做类型上的转接。 + cloudId: record.cloudId as unknown as string, + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + md5: record.md5 as string, + embedded: Boolean(record.embedded), + ...parseJson>(record.extra, {}), + }) + }, +} + +export class SqlChannelStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlChannelStorage' + } + + async setItem(id: string, channel: Channel): Promise { + stampTimestamps(channel) + return super.setItem(id, channel) + } + + /** + * models 存的是 JSON 数组，按子串匹配单个模型名。 + * + * 改造前这是藏在 listItemsByEqFilter 里的一个 LIKE 特判——叫「等值过滤」却做 + * 子串匹配是误导，这里提成显式方法。 + */ + async listItemsByModel(model: string): Promise { + await this.ensureInitialized() + const rows = await this.driver.all( + `SELECT * FROM ${this.q(this.table)} WHERE ${this.q('models')} LIKE ?`, + [`%${model}%`], + ) + return rows.map(row => spec.fromRecord(row)).filter((item): item is Channel => Boolean(item)) + } +} + +export { spec as channelSpec } diff --git a/src/storage/sql/chat_preset_storage.ts b/src/storage/sql/chat_preset_storage.ts new file mode 100644 index 0000000..d949423 --- /dev/null +++ b/src/storage/sql/chat_preset_storage.ts @@ -0,0 +1,81 @@ +import { ChatPreset } from '../../channels/preset' +import { SendMessageOption } from '../../types' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from '../sql_storage' +import { stampTimestamps } from './shareable' +import { CHAT_PRESETS } from './tables' + +const spec: TableSpec = { + table: CHAT_PRESETS.table, + columns: CHAT_PRESETS.columns, + indexes: [{ columns: ['prefix'] }, { columns: ['name'] }], + filterable: ['id', 'name', 'description', 'prefix', 'namespace', 'cloudId', 'local', 'embedded'], + boolean: ['local', 'embedded'], + toRecord(preset, id) { + const { + id: _ignored, name, description, prefix, local, namespace, + sendMessageOption, cloudId, createdAt, updatedAt, md5, + embedded, uploader, ...rest + } = preset as ChatPreset & Record + + return { + id, + name: name || '', + description: description || '', + prefix: prefix || '', + // 历史行为：只有显式 false 才存 0 + local: local === false ? 0 : 1, + namespace: namespace || null, + sendMessageOption: JSON.stringify(sendMessageOption || {}), + cloudId: cloudId || null, + createdAt: createdAt || '', + updatedAt: updatedAt || '', + md5: md5 || '', + embedded: embedded ? 1 : 0, + uploader: stringifyJson(uploader), + extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null, + } + }, + fromRecord(record) { + if (!record) return null + return new ChatPreset({ + id: record.id as string, + name: record.name as string, + description: record.description as string, + prefix: record.prefix as string, + local: Boolean(record.local), + namespace: record.namespace as string, + sendMessageOption: parseJson(record.sendMessageOption, {}) as SendMessageOption, + cloudId: record.cloudId as unknown as string, + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + md5: record.md5 as string, + embedded: Boolean(record.embedded), + uploader: parseJson(record.uploader, undefined), + ...parseJson>(record.extraData, {}), + }) + }, +} + +export class SqlChatPresetStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlChatPresetStorage' + } + + async setItem(id: string, preset: ChatPreset): Promise { + stampTimestamps(preset) + return super.setItem(id, preset) + } + + async getPresetByPrefix(prefix: string): Promise { + await this.ensureInitialized() + const row = await this.driver.get(`SELECT * FROM ${this.q(this.table)} WHERE ${this.q('prefix')} = ?`, [prefix]) + return row ? spec.fromRecord(row) : null + } +} + +export { spec as chatPresetSpec } diff --git a/src/storage/sql/executable_storage.ts b/src/storage/sql/executable_storage.ts new file mode 100644 index 0000000..d6a56b4 --- /dev/null +++ b/src/storage/sql/executable_storage.ts @@ -0,0 +1,151 @@ +import { ToolDTO } from '../../types/tools' +import { TriggerDTO } from '../../types/trigger' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from '../sql_storage' +import { stampTimestamps } from './shareable' +import { TOOLS, TRIGGERS } from './tables' + +/** + * 工具和触发器的表结构除了 isOneTime 之外完全一样，序列化逻辑也一样。 + * 与其抄两遍，不如把共同部分参数化——这正是整个改造要消灭的那种重复。 + */ +const EXECUTABLE_FILTERABLE = [ + 'id', 'name', 'description', 'modelType', 'cloudId', 'md5', 'status', 'permission', 'embedded', +] + +interface ExecutableLike { + id?: string + name?: string + description?: string + modelType?: string + code?: string + cloudId?: string + createdAt?: string + updatedAt?: string + md5?: string + embedded?: boolean + uploader?: unknown + status?: string + permission?: string +} + +function writeExecutable(entity: ExecutableLike & Record, id: string, defaultModelType: string) { + const { + id: _ignored, name, description, modelType, code, cloudId, + embedded, uploader, createdAt, updatedAt, md5, + status, permission, isOneTime: _isOneTime, ...rest + } = entity + + return { + record: { + id, + name: name || '', + description: description || '', + modelType: modelType || defaultModelType, + code: code || null, + cloudId: cloudId || null, + embedded: embedded ? 1 : 0, + uploader: stringifyJson(uploader), + createdAt: createdAt || '', + updatedAt: updatedAt || '', + md5: md5 || '', + status: status || 'enabled', + permission: permission || 'public', + }, + rest, + } +} + +function readExecutable(record: Record) { + return { + id: record.id as string, + name: record.name as string, + description: record.description as string, + modelType: record.modelType as 'settings' | 'executable' | undefined, + code: record.code as string, + cloudId: record.cloudId as unknown as string, + embedded: Boolean(record.embedded), + uploader: parseJson(record.uploader, undefined), + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + md5: record.md5 as string, + status: record.status as 'enabled' | 'disabled', + permission: record.permission as 'public' | 'private' | 'onetime', + ...parseJson>(record.extraData, {}), + } +} + +const toolsSpec: TableSpec = { + table: TOOLS.table, + columns: TOOLS.columns, + indexes: [ + { columns: ['name'], name: 'idx_tools_name' }, + { columns: ['status'], name: 'idx_tools_status' }, + { columns: ['permission'], name: 'idx_tools_permission' }, + ], + filterable: EXECUTABLE_FILTERABLE, + boolean: ['embedded'], + toRecord(tool, id) { + const { record, rest } = writeExecutable(tool as ToolDTO & Record, id, '') + return { ...record, extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null } + }, + fromRecord(record) { + if (!record) return null + return new ToolDTO(readExecutable(record)) + }, +} + +const triggerSpec: TableSpec = { + table: TRIGGERS.table, + columns: TRIGGERS.columns, + indexes: [ + { columns: ['name'], name: 'idx_triggers_name' }, + { columns: ['status'], name: 'idx_triggers_status' }, + ], + filterable: [...EXECUTABLE_FILTERABLE, 'isOneTime'], + boolean: ['embedded', 'isOneTime'], + toRecord(trigger, id) { + const { record, rest } = writeExecutable(trigger as TriggerDTO & Record, id, 'executable') + return { + ...record, + isOneTime: trigger.isOneTime ? 1 : 0, + extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null, + } + }, + fromRecord(record) { + if (!record) return null + return new TriggerDTO({ ...readExecutable(record), isOneTime: Boolean(record.isOneTime) }) + }, +} + +export class SqlToolsStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, toolsSpec) + } + + getName(): string { + return 'SqlToolsStorage' + } + + async setItem(id: string, tool: ToolDTO): Promise { + stampTimestamps(tool) + return super.setItem(id, tool) + } +} + +export class SqlTriggerStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, triggerSpec) + } + + getName(): string { + return 'SqlTriggerStorage' + } + + async setItem(id: string, trigger: TriggerDTO): Promise { + stampTimestamps(trigger) + return super.setItem(id, trigger) + } +} + +export { toolsSpec, triggerSpec } diff --git a/src/storage/sql/history_manager.ts b/src/storage/sql/history_manager.ts new file mode 100644 index 0000000..bd70a3f --- /dev/null +++ b/src/storage/sql/history_manager.ts @@ -0,0 +1,526 @@ +import path from 'node:path' +import fs from 'node:fs' +import crypto from 'node:crypto' +import { AbstractHistoryManager } from '../../types/adapter' +import { HistoryMessage } from '../../types/models' +import { SqlDriver } from '../driver/types' +import { isBase64Image } from './base64_image' +import { log } from '../logger' +import { HISTORY } from './tables' + +interface HistoryRecord { + id: string + parentId: string | null + conversationId: string + role: string + messageData: string + createdAt: string +} + +/** + * 处理图片时用的宽松视图。MessageContent 是个联合类型，而这里要就地改写 + * item.image，用联合类型会被窄化规则挡住，所以本地放宽再转回去。 + */ +type MutableMessage = Omit & { content: ImageContentItem[] } + +interface ImageContentItem { + type?: string + image?: string + mimeType?: string + text?: string + _type?: string +} + +const MIME_EXTENSIONS: Record = { + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', + 'image/svg+xml': '.svg', +} + +const IMAGE_REF = /^\$image:([a-f0-9]+):(\.[a-z]+)$/ + +/** + * 消息链回溯的深度上限。 + * + * 正常会话远达不到这个数；设上限是为了让一条成环的 parentId 不至于把递归查询 + * 拖死。超过上限会告警并截断。 + */ +const MAX_CHAIN_DEPTH = 10000 + +/** + * 前缀的下一个字符串，用于把「以 prefix 开头」表达成范围比较。 + * + * 用范围而不是 LIKE 'prefix%'：LIKE 能不能走索引取决于 case_sensitive_like + * 之类的设置，范围比较则一定能用上 conversationId 索引。 + */ +function nextPrefix(prefix: string): string { + const last = prefix.charCodeAt(prefix.length - 1) + return prefix.slice(0, -1) + String.fromCharCode(last + 1) +} + +export interface PruneOptions { + before: string + conversationPrefix?: string + batchSize?: number + maxBatches?: number +} + +/** + * 基于 SQL 的对话历史。 + * + * 和其他 storage 不同，它实现的是 AbstractHistoryManager 而不是 ChaiteStorage， + * 而且还要把消息里的 base64 图片落盘换成引用——历史库是最大的一张表，把几 MB 的 + * 图片塞进行里会让它迅速膨胀到几百 MB。 + */ +export class SqlHistoryManager extends AbstractHistoryManager { + name = 'SqlHistoryManager' + + readonly driver: SqlDriver + readonly imagesDir: string + readonly table = 'history' + + /** + * 保留期清理按 createdAt 过滤，没有索引就是全表扫描。但在已经很大的历史表上 + * 建索引可能要好几秒，不该卡住升级后的第一条消息，所以放到后台用 low 优先级 + * 建——实时对话的写入会插在前面，建好之前清理只是慢一点，不影响正确性。 + * 暴露成 promise，便于测试和清理流程在需要时等待。 + */ + createdAtIndexReady?: Promise + + private initialized = false + private initPromise: Promise | null = null + + constructor(driver: SqlDriver, imagesDir: string) { + super() + this.driver = driver + this.imagesDir = imagesDir + } + + private q(name: string): string { + return this.driver.dialect.quoteId(name) + } + + async initialize(): Promise { + if (this.initialized) return + if (this.initPromise) return this.initPromise + + this.initPromise = (async () => { + const dialect = this.driver.dialect + fs.mkdirSync(this.imagesDir, { recursive: true }) + + await this.driver.exec( + dialect.createTable(this.table, HISTORY.columns), + ) + await this.driver.exec( + dialect.createIndex(this.table, ['conversationId'], { name: 'idx_history_conversation' }), + ) + await this.driver.exec(dialect.createIndex(this.table, ['parentId'], { name: 'idx_history_parent' })) + this.initialized = true + + this.createdAtIndexReady = this.driver + .exec(dialect.createIndex(this.table, ['createdAt'], { name: 'idx_history_created' }), { + priority: 'low', + label: 'create createdAt index', + }) + .catch((error: unknown) => { + log().warn( + `[History] failed to create createdAt index: ${error instanceof Error ? error.message : String(error)}`, + ) + }) + })().catch((error: unknown) => { + this.initPromise = null + throw error + }) + + return this.initPromise + } + + async ensureInitialized(): Promise { + if (!this.initialized) await this.initialize() + } + + private getMd5(text: string): string { + return crypto.createHash('md5').update(text).digest('hex') + } + + private getMimeTypeFromBase64(base64: string, defaultMimeType = 'image/jpeg'): string { + if (base64 && base64.startsWith('data:image/')) { + const match = base64.match(/^data:(image\/[a-zA-Z+]+);base64,/) + if (match) return match[1] + } + // 纯 base64 字符串没有类型信息，用调用方给的默认值 + return defaultMimeType + } + + private getExtensionFromMimeType(mimeType: string): string { + return MIME_EXTENSIONS[mimeType] || '.png' + } + + /** 把消息里的 base64 图片落盘，行里只留 `$image:md5:ext` 引用。 */ + private processMessageImages(message: HistoryMessage): HistoryMessage { + if (!message.content || !Array.isArray(message.content)) return message + + const processed = JSON.parse(JSON.stringify(message)) as MutableMessage + processed.content = processed.content.map((item: ImageContentItem) => { + if (item.type !== 'image' || !item.image || !isBase64Image(item.image)) return item + + let base64Data = item.image + let mimeType = item.mimeType || 'image/jpeg' + + if (base64Data.startsWith('data:')) { + const parts = base64Data.split(',') + if (parts.length > 1) { + base64Data = parts[1] + mimeType = this.getMimeTypeFromBase64(item.image, mimeType) + } + } + + try { + const md5 = this.getMd5(base64Data) + const ext = this.getExtensionFromMimeType(mimeType) + const filePath = path.join(this.imagesDir, `${md5}${ext}`) + // 同一张图只落一次盘 + if (!fs.existsSync(filePath)) { + fs.writeFileSync(filePath, Buffer.from(base64Data, 'base64')) + } + item.image = `$image:${md5}:${ext}` + item._type = mimeType + } catch (error) { + log().warn(`[History] 保存图片失败: ${error instanceof Error ? error.message : String(error)}`) + } + return item + }) + + return processed as unknown as HistoryMessage + } + + /** 把 `$image:md5:ext` 引用读回 base64；文件没了就把这一项去掉并补上 [图片]。 */ + private restoreMessageImages(message: HistoryMessage): HistoryMessage { + if (!message || !message.content || !Array.isArray(message.content)) return message + + const restored = JSON.parse(JSON.stringify(message)) as MutableMessage + let needImageText = true + let hasRemovedImage = false + + restored.content = restored.content.filter((item: ImageContentItem) => { + if (item.type === 'image' && typeof item.image === 'string') { + const match = item.image.match(IMAGE_REF) + if (match) { + const filePath = path.join(this.imagesDir, `${match[1]}${match[2]}`) + if (!fs.existsSync(filePath)) { + hasRemovedImage = true + return false + } + try { + item.image = fs.readFileSync(filePath).toString('base64') + return true + } catch (error) { + log().warn( + `[History] 读取图片文件失败 ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + ) + hasRemovedImage = true + return false + } + } + } + if (item.type === 'text') needImageText = false + return true + }) + + if (hasRemovedImage) { + if (restored.content.length === 0) { + restored.content.push({ type: 'text', text: '[图片]' }) + } else if (needImageText) { + const textIndex = restored.content.findIndex((item: ImageContentItem) => item.type === 'text') + if (textIndex !== -1) { + restored.content[textIndex].text = `[图片] ${restored.content[textIndex].text}` + } else { + restored.content.unshift({ type: 'text', text: '[图片]' }) + } + } + } + + return restored as unknown as HistoryMessage + } + + private messageToRecord(message: HistoryMessage, conversationId: string): HistoryRecord { + const processed = this.processMessageImages(message) + const { id, parentId, role } = processed as HistoryMessage & { parentId?: string } + return { + id: id || '', + parentId: parentId || null, + conversationId: conversationId || '', + role: role || '', + messageData: JSON.stringify(processed), + createdAt: new Date().toISOString(), + } + } + + private recordToMessage(record: Record | undefined): HistoryMessage | null { + if (!record) return null + try { + return this.restoreMessageImages(JSON.parse(String(record.messageData)) as HistoryMessage) + } catch { + // 解析失败就尽量给个最小结构，别让整个会话读不出来 + return { + id: record.id, + parentId: record.parentId, + role: record.role, + conversationId: record.conversationId, + content: [], + } as unknown as HistoryMessage + } + } + + private upsertSql(): string { + const fields = ['id', 'parentId', 'conversationId', 'role', 'messageData', 'createdAt'] + return this.driver.dialect.upsert(this.table, fields, 'id') + } + + private recordValues(record: HistoryRecord): unknown[] { + return [record.id, record.parentId, record.conversationId, record.role, record.messageData, record.createdAt] + } + + async saveHistory(message: HistoryMessage, conversationId: string): Promise { + await this.ensureInitialized() + const record = this.messageToRecord(message, conversationId) + await this.driver.run(this.upsertSql(), this.recordValues(record), { + priority: 'high', + label: 'save history message', + }) + } + + /** 批量保存走一个事务，避免多条消息逐条 autocommit。 */ + async saveHistories(messages: HistoryMessage[], conversationId: string): Promise { + await this.ensureInitialized() + if (!messages || messages.length === 0) return + + const records = messages.map(message => this.messageToRecord(message, conversationId)) + const sql = this.upsertSql() + await this.driver.transaction( + async transaction => { + for (const record of records) { + await transaction.run(sql, this.recordValues(record)) + } + }, + { priority: 'high', label: `save ${records.length} history messages` }, + ) + } + + async getHistory(messageId?: string, conversationId?: string): Promise { + await this.ensureInitialized() + if (messageId) return this.getMessageChain(messageId) + if (conversationId) return this.getConversationMessages(conversationId) + return [] + } + + /** + * 从指定消息沿 parentId 追溯到根。 + * + * 用递归 CTE 一次查完，而不是每条消息发一次查询。原来那种逐条回溯是典型的 + * N+1：链有多长就要多少次往返。本机 SQLite 上一次往返约 0.022ms，一条 3000 + * 条的链就要 65ms；换成走网络的 Postgres，按单次往返 0.3ms 算会变成近 1 秒。 + * 而且这是每条消息进对话时都要做一次的事。 + * + * 递归 CTE 两种方言都支持（SQLite 3.8.3+、Postgres 一直有）。 + * + * depth 上限同时兼作防环措施：parentId 理论上不该成环，但一条坏数据就足以 + * 让递归停不下来。 + */ + private async getMessageChain(messageId: string): Promise { + const table = this.q(this.table) + const id = this.q('id') + const parentId = this.q('parentId') + + const rows = await this.driver.all>( + `WITH RECURSIVE ${this.q('chain')} AS ( + SELECT ${table}.*, 0 AS ${this.q('depth')} FROM ${table} WHERE ${id} = ? + UNION ALL + SELECT ${table}.*, ${this.q('chain')}.${this.q('depth')} + 1 + FROM ${table} + JOIN ${this.q('chain')} ON ${table}.${id} = ${this.q('chain')}.${parentId} + WHERE ${this.q('chain')}.${this.q('depth')} < ? + ) + SELECT * FROM ${this.q('chain')} ORDER BY ${this.q('depth')} ASC`, + [messageId, MAX_CHAIN_DEPTH], + ) + + // 按 depth 升序拿回来（目标消息 -> 父 -> 祖父 …），遇到重复 id 就停： + // 这和改造前逐条回溯 + visited 集合的语义完全一致。深度上限只保证成环时 + // 递归一定会停，真正判环还是靠这里。 + const chain: HistoryMessage[] = [] + const seen = new Set() + for (const row of rows) { + const id = String(row.id) + if (seen.has(id)) { + log().warn(`[History] detected a parentId cycle at ${id}, truncating the chain`) + break + } + seen.add(id) + const message = this.recordToMessage(row) + if (message) chain.push(message) + } + + if (rows.length >= MAX_CHAIN_DEPTH) { + log().warn(`[History] chain from ${messageId} hit the depth cap (${MAX_CHAIN_DEPTH}) and was truncated`) + } + + // 调用方要的是从根到目标的顺序 + return chain.reverse() + } + + private async getConversationMessages(conversationId: string): Promise { + const rows = await this.driver.all( + `SELECT * FROM ${this.q(this.table)} WHERE ${this.q('conversationId')} = ? ORDER BY ${this.q('createdAt')}`, + [conversationId], + ) + return rows.map(row => this.recordToMessage(row)).filter((m): m is HistoryMessage => Boolean(m)) + } + + async deleteConversation(conversationId: string): Promise { + await this.ensureInitialized() + await this.driver.run(`DELETE FROM ${this.q(this.table)} WHERE ${this.q('conversationId')} = ?`, [conversationId]) + } + + /** 删一条消息但不断链：先把它的直接子节点改挂到它的父节点上。 */ + async removeHistory(messageId: string, conversationId: string): Promise { + await this.ensureInitialized() + await this.driver.transaction( + async transaction => { + const row = await transaction.get<{ parentId: string | null }>( + `SELECT ${this.q('parentId')} FROM ${this.q(this.table)} + WHERE ${this.q('id')} = ? AND ${this.q('conversationId')} = ?`, + [messageId, conversationId], + ) + if (!row) return + await transaction.run( + `UPDATE ${this.q(this.table)} SET ${this.q('parentId')} = ? + WHERE ${this.q('conversationId')} = ? AND ${this.q('parentId')} = ?`, + [row.parentId || null, conversationId, messageId], + ) + await transaction.run( + `DELETE FROM ${this.q(this.table)} WHERE ${this.q('id')} = ? AND ${this.q('conversationId')} = ?`, + [messageId, conversationId], + ) + }, + { priority: 'high', label: 'remove history message' }, + ) + } + + async getOneHistory(messageId: string, conversationId: string): Promise { + await this.ensureInitialized() + const conditions: string[] = [] + const params: unknown[] = [] + + if (messageId) { + conditions.push(`${this.q('id')} = ?`) + params.push(messageId) + } + if (conversationId) { + conditions.push(`${this.q('conversationId')} = ?`) + params.push(conversationId) + } + if (conditions.length === 0) return undefined + + const row = await this.driver.get( + `SELECT * FROM ${this.q(this.table)} WHERE ${conditions.join(' AND ')} LIMIT 1`, + params, + ) + return this.recordToMessage(row) ?? undefined + } + + private buildPruneFilter(before: string, conversationPrefix: string) { + const conditions = [`${this.q('createdAt')} < ?`] + const params: unknown[] = [before] + if (conversationPrefix) { + conditions.push(`${this.q('conversationId')} >= ?`, `${this.q('conversationId')} < ?`) + params.push(conversationPrefix, nextPrefix(conversationPrefix)) + } + return { where: conditions.join(' AND '), params } + } + + async countHistoryBefore({ before, conversationPrefix = '' }: { before: string, conversationPrefix?: string }): Promise { + await this.ensureInitialized() + if (!before) return 0 + const { where, params } = this.buildPruneFilter(before, conversationPrefix) + const row = await this.driver.get<{ count: number }>( + `SELECT COUNT(*) AS "count" FROM ${this.q(this.table)} WHERE ${where}`, + params, + ) + return Number(row?.count || 0) + } + + /** + * 按保留期删除历史消息。 + * + * 分批删而不是一条 DELETE 干掉几十万行：单条大 DELETE 会长时间占住写连接， + * 期间所有对话的写入都要排队。批次走 low 优先级，让实时对话的写入插在前面。 + */ + async pruneHistory({ + before, + conversationPrefix = '', + batchSize = 2000, + maxBatches = 500, + }: PruneOptions): Promise<{ deleted: number, truncated: boolean }> { + await this.ensureInitialized() + if (!before) return { deleted: 0, truncated: false } + + const { where, params } = this.buildPruneFilter(before, conversationPrefix) + const sql = `DELETE FROM ${this.q(this.table)} WHERE ${this.q('id')} IN ( + SELECT ${this.q('id')} FROM ${this.q(this.table)} WHERE ${where} LIMIT ? + )` + + let deleted = 0 + for (let batch = 0; batch < maxBatches; batch++) { + const result = await this.driver.run(sql, [...params, batchSize], { + priority: 'low', + label: 'prune history', + }) + const changes = result?.changes || 0 + deleted += changes + if (changes < batchSize) return { deleted, truncated: false } + } + // 还没删完，剩下的留给下一次，避免一次任务跑太久 + return { deleted, truncated: true } + } + + /** 删掉已经没有任何消息引用的图片文件。 */ + async cleanupUnusedImages(): Promise<{ deleted: number, total: number }> { + await this.ensureInitialized() + const rows = await this.driver.all<{ messageData: string }>( + `SELECT ${this.q('messageData')} FROM ${this.q(this.table)}`, + [], + ) + + const usedImageRefs = new Set() + for (const row of rows) { + try { + const message = JSON.parse(row.messageData) as { content?: ImageContentItem[] } + if (!Array.isArray(message.content)) continue + for (const item of message.content) { + if (item.type === 'image' && typeof item.image === 'string') { + const match = item.image.match(IMAGE_REF) + if (match) usedImageRefs.add(`${match[1]}${match[2]}`) + } + } + } catch { + // 单条坏数据不该让整次清理失败 + } + } + + const files = fs.readdirSync(this.imagesDir) + let deleted = 0 + for (const file of files) { + if (!usedImageRefs.has(file)) { + fs.unlinkSync(path.join(this.imagesDir, file)) + deleted++ + } + } + return { deleted, total: files.length } + } + + /** 连接归 DriverRegistry 管，这里不关。 */ + async close(): Promise {} +} diff --git a/src/storage/sql/mcp_server_storage.ts b/src/storage/sql/mcp_server_storage.ts new file mode 100644 index 0000000..45d7829 --- /dev/null +++ b/src/storage/sql/mcp_server_storage.ts @@ -0,0 +1,42 @@ +import { McpServerConfig } from '../../agent/mcp/McpServerConfig' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson } from '../sql_storage' +import { MCP_SERVERS } from './tables' + +/** + * MCP 连接配置。里面有凭据（headers 里的 Authorization、env 里的 API key）， + * 所以整条记录只落本地库，不上传云端。 + */ +const spec: TableSpec = { + table: MCP_SERVERS.table, + columns: MCP_SERVERS.columns, + indexes: [{ columns: ['name'], name: 'idx_mcp_servers_name' }], + filterable: ['id', 'name', 'enabled'], + boolean: ['enabled'], + orderBy: 'updatedAt DESC', + toRecord(value, id) { + return { + id, + name: value.name, + enabled: value.enabled ? 1 : 0, + updatedAt: value.updatedAt || Date.now(), + payload: JSON.stringify(value), + } + }, + fromRecord(record) { + if (!record) return null + return parseJson(record.payload, null) + }, +} + +export class SqlMcpServerStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlMcpServerStorage' + } +} + +export { spec as mcpServerSpec } diff --git a/src/storage/sql/operation_log_storage.ts b/src/storage/sql/operation_log_storage.ts new file mode 100644 index 0000000..c85bd9c --- /dev/null +++ b/src/storage/sql/operation_log_storage.ts @@ -0,0 +1,164 @@ +import { OperationLog } from '../../types/operation-log' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson } from '../sql_storage' +import { log } from '../logger' +import { OPERATION_LOGS } from './tables' + +/** 能下推到 SQL 的列。其余字段都在 payload 里，只能读出来再过滤。 */ +const FILTERABLE = ['id', 'timestamp', 'type', 'level', 'userId', 'groupId', 'channelId', 'model', 'presetId'] + +const spec: TableSpec = { + table: OPERATION_LOGS.table, + columns: OPERATION_LOGS.columns, + indexes: [ + { columns: [{ column: 'timestamp', order: 'DESC' }], name: 'idx_operation_logs_time' }, + { columns: ['userId', { column: 'timestamp', order: 'DESC' }], name: 'idx_operation_logs_user' }, + { columns: ['groupId', { column: 'timestamp', order: 'DESC' }], name: 'idx_operation_logs_group' }, + { columns: ['channelId', { column: 'timestamp', order: 'DESC' }], name: 'idx_operation_logs_channel' }, + { columns: ['type', { column: 'timestamp', order: 'DESC' }], name: 'idx_operation_logs_type' }, + { + columns: [{ column: 'timestamp', order: 'ASC' }, { column: 'id', order: 'ASC' }], + name: 'idx_operation_logs_retention', + }, + ], + filterable: FILTERABLE, + numeric: ['timestamp'], + orderBy: 'timestamp DESC', + toRecord(value, id) { + return { + id, + timestamp: value.timestamp, + type: value.type, + level: value.level, + userId: value.userId ?? null, + groupId: value.groupId ?? null, + channelId: value.channelId ?? null, + model: value.model ?? null, + presetId: value.presetId ?? null, + payload: JSON.stringify(value), + } + }, + fromRecord(record) { + if (!record) return null + return parseJson(record.payload, null) + }, +} + +/** + * 操作日志。 + * + * 和其他 storage 的区别是它有保留期：写入频率高、条数按万计，所以要按 maxEntries + * 滚动裁剪。裁剪分批进行并走低优先级队列，免得在低配机器上把实时对话的写入挤掉。 + */ +export class SqlOperationLogStorage extends SqlKvStorage { + private writeCount = 0 + private maxEntries: number + private prunePromise: Promise | null = null + private readonly pruneBatchSize = 500 + + constructor(driver: SqlDriver, maxEntries = 50000) { + super(driver, spec) + this.maxEntries = normalizeLimit(maxEntries) + } + + getName(): string { + return 'SqlOperationLogStorage' + } + + async initialize(): Promise { + if (this.initialized) return + await super.initialize() + void this.schedulePrune() + } + + async setItem(id: string, value: OperationLog): Promise { + const key = await super.setItem(id, value) + this.writeCount++ + // 低配机器上别每写一条就查一次总数：按比例间隔触发，上限 200 + const pruneEvery = Math.min(200, Math.max(1, Math.floor(this.maxEntries / 10))) + if (this.maxEntries > 0 && this.writeCount % pruneEvery === 0) { + void this.schedulePrune() + } + return key + } + + /** clear 也走分批，避免一次性删掉几十万行把写锁占满。 */ + async clear(): Promise { + await this.ensureInitialized() + for (;;) { + const result = await this.driver.run( + `DELETE FROM ${this.q(this.table)} WHERE ${this.q('id')} IN ( + SELECT ${this.q('id')} FROM ${this.q(this.table)} + ORDER BY ${this.q('timestamp')} ASC, ${this.q('id')} ASC LIMIT ? + )`, + [this.pruneBatchSize], + { priority: 'low', label: 'clear operation logs batch' }, + ) + if (result.changes < this.pruneBatchSize) return + await new Promise(resolve => setImmediate(resolve)) + } + } + + async setMaxEntries(maxEntries: number): Promise { + this.maxEntries = normalizeLimit(maxEntries) + await (this.schedulePrune(true) || Promise.resolve()) + } + + private schedulePrune(force = false): Promise | null { + if (this.prunePromise) { + // force 时要等当前这轮跑完再按新的 maxEntries 重来一次 + return force ? this.prunePromise.then(() => this.schedulePrune(true) ?? undefined) : this.prunePromise + } + if (this.maxEntries <= 0) return null + const targetEntries = this.maxEntries + this.prunePromise = this.prune(targetEntries, force) + .catch((error: unknown) => + log().warn( + `[operation_logs] prune failed: ${error instanceof Error ? error.message : String(error)}`, + ), + ) + .finally(() => { + this.prunePromise = null + }) + return this.prunePromise + } + + private async prune(targetEntries: number, force = false): Promise { + if (targetEntries <= 0) return + await this.ensureInitialized() + // 留 10% 余量，避免刚好卡在阈值时每写一条都要删一条 + const highWatermark = Math.max(targetEntries + 1, Math.ceil(targetEntries * 1.1)) + const row = await this.driver.get<{ count: number }>( + `SELECT COUNT(*) AS "count" FROM ${this.q(this.table)}`, + [], + ) + let remaining = Number(row?.count || 0) + if (!force && remaining <= highWatermark) return + + while (remaining > targetEntries && this.maxEntries > 0) { + const batchSize = Math.min(this.pruneBatchSize, remaining - targetEntries) + const startedAt = Date.now() + const result = await this.driver.run( + `DELETE FROM ${this.q(this.table)} WHERE ${this.q('id')} IN ( + SELECT ${this.q('id')} FROM ${this.q(this.table)} + ORDER BY ${this.q('timestamp')} ASC, ${this.q('id')} ASC LIMIT ? + )`, + [batchSize], + { priority: 'low', label: 'operation log retention batch' }, + ) + remaining -= result.changes + log().debug( + `[operation_logs] pruned ${result.changes} rows in ${Date.now() - startedAt}ms, remaining=${remaining}`, + ) + if (result.changes === 0) break + await new Promise(resolve => setImmediate(resolve)) + } + } +} + +function normalizeLimit(value: number): number { + const limit = Number.parseInt(String(value), 10) + return Number.isFinite(limit) && limit > 0 ? limit : 0 +} + +export { spec as operationLogSpec } diff --git a/src/storage/sql/processors_storage.ts b/src/storage/sql/processors_storage.ts new file mode 100644 index 0000000..23bef6e --- /dev/null +++ b/src/storage/sql/processors_storage.ts @@ -0,0 +1,69 @@ +import { ProcessorDTO } from '../../types/processors' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from '../sql_storage' +import { stampTimestamps } from './shareable' +import { PROCESSORS } from './tables' + +const spec: TableSpec = { + table: PROCESSORS.table, + columns: PROCESSORS.columns, + indexes: [{ columns: ['type'] }], + filterable: ['id', 'name', 'description', 'type', 'cloudId', 'md5', 'embedded'], + boolean: ['embedded'], + toRecord(processor, id) { + const { + id: _ignored, name, description, type, code, cloudId, + createdAt, updatedAt, md5, embedded, uploader, ...rest + } = processor as ProcessorDTO & Record + + return { + id, + name: name || '', + description: description || '', + // 'pre' 或 'post' + type: type || '', + code: code || '', + cloudId: cloudId || null, + createdAt: createdAt || '', + updatedAt: updatedAt || '', + md5: md5 || '', + embedded: embedded ? 1 : 0, + uploader: stringifyJson(uploader), + extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null, + } + }, + fromRecord(record) { + if (!record) return null + return new ProcessorDTO({ + id: record.id as string, + name: record.name as string, + description: record.description as string, + type: record.type as ProcessorDTO['type'], + code: record.code as string, + cloudId: record.cloudId as unknown as string, + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + md5: record.md5 as string, + embedded: Boolean(record.embedded), + uploader: parseJson(record.uploader, undefined), + ...parseJson>(record.extraData, {}), + }) + }, +} + +export class SqlProcessorsStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlProcessorsStorage' + } + + async setItem(id: string, processor: ProcessorDTO): Promise { + stampTimestamps(processor) + return super.setItem(id, processor) + } +} + +export { spec as processorsSpec } diff --git a/src/storage/sql/shareable.ts b/src/storage/sql/shareable.ts new file mode 100644 index 0000000..49b3b8d --- /dev/null +++ b/src/storage/sql/shareable.ts @@ -0,0 +1,48 @@ +import { ColumnSpec, ColumnTypeName } from '../driver/types' + +/** + * 绝大多数可分享实体（渠道、预设、工具、处理器、触发器）都继承 AbstractShareable， + * 表结构里因此有一整组完全相同的列。抽出来避免五份拷贝各自漂移。 + */ +export const SHAREABLE_COLUMNS: Record = { + cloudId: { type: 'int' }, + createdAt: { type: 'text' }, + updatedAt: { type: 'text' }, + md5: { type: 'text' }, + embedded: { type: 'bool', default: 0 }, + uploader: { type: 'json' }, +} + +/** 写入时统一打时间戳：首次写入补 createdAt，每次写入刷新 updatedAt。 */ +export function stampTimestamps(entity: T): T { + if (!entity.createdAt) entity.createdAt = new Date().toISOString() + entity.updatedAt = new Date().toISOString() + return entity +} + +/** 把 record 里的公共字段读回实体形状。 */ +export function readShareable(record: Record) { + return { + id: record.id as string, + name: record.name as string, + description: record.description as string, + cloudId: record.cloudId as number, + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + md5: record.md5 as string, + embedded: Boolean(record.embedded), + } +} + +/** 把实体的公共字段写成 record 形状。 */ +export function writeShareable(entity: Record) { + return { + name: (entity.name as string) || '', + description: (entity.description as string) || '', + cloudId: (entity.cloudId as number) || null, + createdAt: (entity.createdAt as string) || '', + updatedAt: (entity.updatedAt as string) || '', + md5: (entity.md5 as string) || '', + embedded: entity.embedded ? 1 : 0, + } +} diff --git a/src/storage/sql/tables.ts b/src/storage/sql/tables.ts new file mode 100644 index 0000000..94201b0 --- /dev/null +++ b/src/storage/sql/tables.ts @@ -0,0 +1,200 @@ +import { ColumnSpec, ColumnTypeName, IndexColumn, IndexOptions, LogicalDatabase } from '../driver/types' + +/** + * 表的形状：表名、列、主键、所属逻辑库。 + * + * 刻意和序列化逻辑分开放。toRecord/fromRecord 要 import DTO 类，而那条链会经由 + * types -> core -> 包入口绕回 storage，形成模块环；迁移工具只需要知道表长什么样， + * 没有理由被卷进去。分开之后这里也成了列定义的唯一出处，storage 和迁移工具各取 + * 所需，不会各写一份然后慢慢漂移。 + */ + +export interface TableShape { + database: LogicalDatabase + table: string + keyColumn: string + columns: Record + /** + * upsert 依赖的唯一约束。 + * + * 只有 keyColumn 不是主键时才需要——user_states 的业务主键是 userId，而 id + * 是代理主键，ON CONFLICT(userId) 必须有唯一索引兜底。迁移工具建完表也要照建， + * 否则往新库里搬数据时会直接报「no unique or exclusion constraint matching + * the ON CONFLICT specification」。 + */ + uniqueIndex?: IndexOptions & { columns: IndexColumn[] } +} + +/** 可分享实体共有的那组列（渠道、预设、工具、处理器、触发器）。 */ +export const SHAREABLE_COLUMNS: Record = { + cloudId: { type: 'int' }, + createdAt: { type: 'text' }, + updatedAt: { type: 'text' }, + md5: { type: 'text' }, + embedded: { type: 'bool', default: 0 }, + uploader: { type: 'json' }, +} + +/** 工具和触发器除 isOneTime 外结构完全一致。 */ +const EXECUTABLE_COLUMNS: Record = { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + description: { type: 'text' }, + modelType: { type: 'text' }, + code: { type: 'text' }, + status: { type: 'text' }, + permission: { type: 'text' }, + ...SHAREABLE_COLUMNS, +} + +export const CHANNELS: TableShape = { + database: 'main', + table: 'channels', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + description: { type: 'text' }, + adapterType: { type: 'text', notNull: true }, + type: { type: 'text', notNull: true }, + weight: { type: 'int', default: 1 }, + priority: { type: 'int', default: 0 }, + status: { type: 'text', default: 'enabled' }, + disabledReason: { type: 'text' }, + models: { type: 'json' }, + options: { type: 'json' }, + statistics: { type: 'json' }, + ...SHAREABLE_COLUMNS, + extra: { type: 'json' }, + }, +} + +export const CHAT_PRESETS: TableShape = { + database: 'main', + table: 'chat_presets', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + description: { type: 'text' }, + prefix: { type: 'text', notNull: true }, + local: { type: 'bool', default: 1 }, + namespace: { type: 'text' }, + sendMessageOption: { type: 'json', notNull: true }, + ...SHAREABLE_COLUMNS, + extraData: { type: 'json' }, + }, +} + +export const TOOLS: TableShape = { + database: 'main', + table: 'tools', + keyColumn: 'id', + columns: { ...EXECUTABLE_COLUMNS, extraData: { type: 'json' } }, +} + +export const TRIGGERS: TableShape = { + database: 'main', + table: 'triggers', + keyColumn: 'id', + columns: { ...EXECUTABLE_COLUMNS, isOneTime: { type: 'bool' }, extraData: { type: 'json' } }, +} + +export const PROCESSORS: TableShape = { + database: 'main', + table: 'processors', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + description: { type: 'text' }, + type: { type: 'text', notNull: true }, + code: { type: 'text' }, + ...SHAREABLE_COLUMNS, + extraData: { type: 'json' }, + }, +} + +export const TOOLS_GROUPS: TableShape = { + database: 'main', + table: 'tools_groups', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + description: { type: 'text' }, + toolIds: { type: 'json', notNull: true }, + isDefault: { type: 'bool', default: 0 }, + createdAt: { type: 'text' }, + updatedAt: { type: 'text' }, + }, +} + +export const USER_STATES: TableShape = { + database: 'main', + table: 'user_states', + // 业务主键是 userId，id 只是个随机 UUID 代理键 + keyColumn: 'userId', + uniqueIndex: { columns: ['userId'], name: 'uniq_user_states_userId', unique: true }, + columns: { + id: { type: 'text', pk: true }, + userId: { type: 'text', notNull: true }, + nickname: { type: 'text' }, + card: { type: 'text' }, + conversations: { type: 'json', notNull: true }, + settings: { type: 'json', notNull: true }, + current: { type: 'json', notNull: true }, + updatedAt: { type: 'bigint' }, + }, +} + +export const MCP_SERVERS: TableShape = { + database: 'main', + table: 'mcp_servers', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + enabled: { type: 'bool', notNull: true, default: 1 }, + updatedAt: { type: 'bigint', notNull: true }, + payload: { type: 'json', notNull: true }, + }, +} + +export const HISTORY: TableShape = { + database: 'history', + table: 'history', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + parentId: { type: 'text' }, + conversationId: { type: 'text' }, + role: { type: 'text' }, + messageData: { type: 'json' }, + createdAt: { type: 'text' }, + }, +} + +export const OPERATION_LOGS: TableShape = { + database: 'operation_logs', + table: 'operation_logs', + keyColumn: 'id', + columns: { + id: { type: 'text', pk: true }, + timestamp: { type: 'bigint', notNull: true }, + type: { type: 'text', notNull: true }, + level: { type: 'text', notNull: true }, + userId: { type: 'text' }, + groupId: { type: 'text' }, + channelId: { type: 'text' }, + model: { type: 'text' }, + presetId: { type: 'text' }, + payload: { type: 'json', notNull: true }, + }, +} + +/** chaite 自己管的全部表。宿主有额外的表可以自己追加。 */ +export const CHAITE_TABLES: TableShape[] = [ + CHANNELS, CHAT_PRESETS, TOOLS, TRIGGERS, PROCESSORS, + TOOLS_GROUPS, USER_STATES, MCP_SERVERS, HISTORY, OPERATION_LOGS, +] diff --git a/src/storage/sql/tool_groups_storage.ts b/src/storage/sql/tool_groups_storage.ts new file mode 100644 index 0000000..40223cc --- /dev/null +++ b/src/storage/sql/tool_groups_storage.ts @@ -0,0 +1,54 @@ +import { ToolsGroupDTO } from '../../types/tools' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson } from '../sql_storage' +import { stampTimestamps } from './shareable' +import { TOOLS_GROUPS } from './tables' + +const spec: TableSpec = { + table: TOOLS_GROUPS.table, + columns: TOOLS_GROUPS.columns, + indexes: [{ columns: ['name'], name: 'idx_tools_groups_name' }], + filterable: ['id', 'name', 'description', 'isDefault'], + boolean: ['isDefault'], + toRecord(group, id) { + const { name, description, toolIds, isDefault, createdAt, updatedAt } = group + return { + id, + name: name || '', + description: description || '', + toolIds: JSON.stringify(toolIds || []), + isDefault: isDefault ? 1 : 0, + createdAt: createdAt || '', + updatedAt: updatedAt || '', + } + }, + fromRecord(record) { + if (!record) return null + return new ToolsGroupDTO({ + id: record.id as string, + name: record.name as string, + description: record.description as string, + toolIds: parseJson(record.toolIds, []), + isDefault: Boolean(record.isDefault), + createdAt: record.createdAt as string, + updatedAt: record.updatedAt as string, + }) + }, +} + +export class SqlToolsGroupStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlToolsGroupStorage' + } + + async setItem(id: string, group: ToolsGroupDTO): Promise { + stampTimestamps(group) + return super.setItem(id, group) + } +} + +export { spec as toolsGroupSpec } diff --git a/src/storage/sql/user_state_storage.ts b/src/storage/sql/user_state_storage.ts new file mode 100644 index 0000000..f747a75 --- /dev/null +++ b/src/storage/sql/user_state_storage.ts @@ -0,0 +1,106 @@ +import * as crypto from 'node:crypto' +import { Conversation, UserSettings, UserState } from '../../types/storage' +import { SqlDriver } from '../driver/types' +import { SqlKvStorage, TableSpec, parseJson } from '../sql_storage' +import { log } from '../logger' +import { USER_STATES } from './tables' + +/** + * user_states 的业务主键是 userId，不是 id——id 只是个随机 UUID 代理键。 + * 所以 keyColumn 指到 userId，并把 id 标成不可覆盖：同一个用户反复写入时 + * 代理键不应该跟着变。 + */ +const spec: TableSpec = { + table: USER_STATES.table, + keyColumn: USER_STATES.keyColumn, + immutableColumns: ['id'], + columns: USER_STATES.columns, + indexes: [{ columns: ['userId'], name: 'idx_user_states_userId' }], + filterable: ['userId', 'nickname', 'card'], + toRecord(userState, userId) { + const { nickname, card, conversations, settings, current } = userState + return { + // 新行才会用到；已存在的行由 immutableColumns 保住原值 + id: crypto.randomUUID(), + userId, + nickname: nickname ?? null, + card: card ?? null, + conversations: JSON.stringify(conversations || []), + settings: JSON.stringify(settings || {}), + current: JSON.stringify(current || {}), + updatedAt: Date.now(), + } + }, + fromRecord(record) { + if (!record) return null + return { + userId: record.userId as string, + nickname: record.nickname as string, + card: record.card as string, + conversations: parseJson(record.conversations, []), + settings: parseJson(record.settings, {} as UserSettings), + current: parseJson(record.current, {}), + } + }, +} + +export class SqlUserStateStorage extends SqlKvStorage { + constructor(driver: SqlDriver) { + super(driver, spec) + } + + getName(): string { + return 'SqlUserStateStorage' + } + + /** + * 在基类建表之外，还要保证 userId 上有唯一索引——ON CONFLICT(userId) 依赖它。 + * + * 老库只有一个普通索引，而且改造前的写入是「先 SELECT 再 INSERT/UPDATE」， + * 两个并发请求给同一个新用户写状态时可以各插一行。所以建唯一索引之前先去重， + * 每个 userId 只留 updatedAt 最新的那行。 + */ + protected async afterInitialize(): Promise { + // 去重和建唯一索引必须是一个原子步骤：中间一旦放行并发写入，就可能又插进 + // 一行重复 userId，随后 CREATE UNIQUE INDEX 直接失败。放在同一个事务里， + // SQLite 靠 writer 队列串行化，Postgres 靠事务本身。 + await this.driver.transaction(async transaction => { + const table = this.q(this.table) + const duplicates = await transaction.all<{ userId: string, count: number }>( + `SELECT ${this.q('userId')} AS "userId", COUNT(*) AS "count" + FROM ${table} GROUP BY ${this.q('userId')} HAVING COUNT(*) > 1`, + [], + ) + + if (duplicates.length > 0) { + const total = duplicates.reduce((sum, row) => sum + Number(row.count) - 1, 0) + log().warn( + `[Storage] user_states 有 ${duplicates.length} 个 userId 存在重复行，` + + `清理 ${total} 行后建立唯一索引（保留 updatedAt 最新的一行）`, + ) + // COALESCE 是为了绕开 NULL 排序差异：SQLite 的 DESC 把 NULL 放最后， + // Postgres 默认放最前 + await transaction.run( + `DELETE FROM ${table} WHERE ${this.q('id')} NOT IN ( + SELECT ${this.q('id')} FROM ( + SELECT ${this.q('id')}, + ROW_NUMBER() OVER ( + PARTITION BY ${this.q('userId')} + ORDER BY COALESCE(${this.q('updatedAt')}, 0) DESC, ${this.q('id')} DESC + ) AS rn + FROM ${table} + ) ranked WHERE rn = 1 + )`, + [], + ) + } + + await transaction.run( + this.dialect.createIndex(this.table, ['userId'], { name: 'uniq_user_states_userId', unique: true }), + [], + ) + }, { priority: 'high', label: 'user_states unique index' }) + } +} + +export { spec as userStateSpec } diff --git a/src/storage/sql_storage.ts b/src/storage/sql_storage.ts new file mode 100644 index 0000000..bb38669 --- /dev/null +++ b/src/storage/sql_storage.ts @@ -0,0 +1,292 @@ +import { ChaiteStorage } from '../types/storage' +import { ColumnSpec, ColumnTypeName, IndexColumn, IndexOptions, SqlDriver } from './driver/types' +import { listColumns } from './driver/introspect' +import { log } from './logger' + +/** + * 和宿主插件里原有的 generateId 保持同样的形状，避免迁移后新旧 id 风格不一致。 + */ +function generateId(): string { + return Date.now().toString(36) + Math.random().toString(36).substring(2, 15) +} + +export interface TableSpec { + /** 表名 */ + table: string + /** 列定义 */ + columns: Record + /** 主键列，默认 'id'。user_states 用 userId 做业务主键 */ + keyColumn?: string + /** 冲突时不覆盖的列，例如代理主键 */ + immutableColumns?: string[] + indexes?: (IndexOptions & { columns: IndexColumn[] })[] + /** 可以下推到 SQL 的列；其余条件在内存里过滤 */ + filterable?: string[] + /** 过滤时需要 Number() 归一的列 */ + numeric?: string[] + /** 过滤时需要归一成 0/1 的列 */ + boolean?: string[] + /** listItems 的排序，如 'updatedAt DESC' */ + orderBy?: string + toRecord(entity: T, id: string): Record + fromRecord(record: Record): T | null +} + +/** + * 「键值表 + 提升列」的公共实现。 + * + * 宿主插件里的九个 storage 在关系意义上都不是关系数据：一个 TEXT 主键、几个为了 + * 能下推过滤而提升出来的列、剩下的塞进 JSON 列，一个 JOIN 都没有。所以这里不引 + * ORM——ORM 的价值在关系映射和查询组合，这套代码一样都用不上。 + * + * 子类只需要给出一份 TableSpec，七个 ChaiteStorage 方法由这里统一实现，SQL 一份， + * 方言差异由 driver 兜住。 + */ +export class SqlKvStorage extends ChaiteStorage { + readonly driver: SqlDriver + readonly spec: TableSpec + readonly table: string + readonly keyColumn: string + + protected initialized = false + private initPromise: Promise | null = null + + constructor(driver: SqlDriver, spec: TableSpec) { + super() + this.driver = driver + this.spec = spec + this.table = spec.table + this.keyColumn = spec.keyColumn || 'id' + } + + get dialect() { + return this.driver.dialect + } + + /** + * 建表与索引。重复调用是安全的。 + * + * **子类不要覆盖这个方法**，要加初始化步骤请覆盖 afterInitialize()。 + * 覆盖 initialize() 并在里面 `await super.initialize()` 会有竞态：基类返回时 + * initialized 已经是 true，此时另一个并发的 ensureInitialized() 会立刻放行， + * 而子类那部分（建唯一索引之类）还没跑完。 + */ + async initialize(): Promise { + if (this.initialized) return + if (this.initPromise) return this.initPromise + + this.initPromise = (async () => { + await this.driver.exec(this.dialect.createTable(this.table, this.spec.columns)) + await this.reconcileColumns() + for (const index of this.spec.indexes || []) { + await this.driver.exec(this.dialect.createIndex(this.table, index.columns, index)) + } + // 子类的补充步骤必须在置位之前完成，否则并发调用会看到一个只做了一半的表 + await this.afterInitialize() + this.initialized = true + })().catch((error: unknown) => { + // 不要把失败的 promise 留在字段里，否则一次偶发失败会让这张表到重启为止 + // 都初始化不了。initialized 也仍然是 false，下次会重试整条链路。 + this.initPromise = null + throw error + }) + + return this.initPromise + } + + /** + * 建表之后、标记就绪之前的钩子。子类在这里做额外的模式调整。 + * 抛错会让整次 initialize 失败并允许重试。 + */ + protected async afterInitialize(): Promise {} + + /** + * 给老表补上新增的列。 + * + * CREATE TABLE IF NOT EXISTS 对已存在的表是空操作，不会补列。于是从更早版本 + * 升上来的库里，表可能缺某几列，而写入路径会把当前全部列都列进 INSERT， + * 运行时直接报 "no column named ..." / "column does not exist"。 + * + * 这里对比实际列和 spec，把缺的 ALTER TABLE ADD COLUMN 补回去。 + * + * 两个做不到的情况会告警而不是硬失败——继续跑总比因为一次模式差异起不来好： + * - 主键列缺失：两种方言都不允许事后加主键，通常意味着这张表根本不是我们的 + * - NOT NULL 且没有默认值：已有行无法填充，只能降级成可空 + */ + private async reconcileColumns(): Promise { + const existing = await listColumns(this.driver, this.table) + if (existing.size === 0) return + + for (const [name, raw] of Object.entries(this.spec.columns)) { + if (existing.has(name)) continue + const opts: ColumnSpec = typeof raw === 'string' ? { type: raw } : raw + + if (opts.pk || opts.autoIncrement) { + log().error( + `[Storage] ${this.table} 缺少主键列 ${name}，无法通过 ALTER TABLE 补上。` + + '这张表大概不是本存储层创建的，请人工核对。', + ) + continue + } + + let definition = `${this.dialect.quoteId(name)} ${this.dialect.columnType(opts.type ?? 'text')}` + if (opts.default !== undefined) { + definition += ` DEFAULT ${typeof opts.default === 'number' ? opts.default : `'${String(opts.default).replace(/'/g, '\'\'')}'`}` + } + if (opts.notNull && opts.default === undefined) { + log().warn( + `[Storage] ${this.table}.${name} 声明为 NOT NULL 但没有默认值，` + + '已有行无法填充，补列时降级为可空', + ) + } else if (opts.notNull) { + definition += ' NOT NULL' + } + + await this.driver.exec(`ALTER TABLE ${this.dialect.quoteId(this.table)} ADD COLUMN ${definition}`) + log().info(`[Storage] ${this.table}: 补上缺失的列 ${name}`) + } + } + + async ensureInitialized(): Promise { + if (!this.initialized) await this.initialize() + } + + protected q(name: string): string { + return this.dialect.quoteId(name) + } + + protected orderClause(): string { + if (!this.spec.orderBy) return '' + const [column, direction = 'ASC'] = this.spec.orderBy.split(/\s+/) + const dir = direction.toUpperCase() === 'DESC' ? 'DESC' : 'ASC' + return ` ORDER BY ${this.q(column)} ${dir}` + } + + async getItem(key: string): Promise { + await this.ensureInitialized() + const row = await this.driver.get(`SELECT * FROM ${this.q(this.table)} WHERE ${this.q(this.keyColumn)} = ?`, [key]) + return row ? this.spec.fromRecord(row) : null + } + + async setItem(id: string, value: T): Promise { + await this.ensureInitialized() + const key = id || generateId() + const record = this.spec.toRecord(value, key) + record[this.keyColumn] = key + + const columns = Object.keys(record) + const sql = this.dialect.upsert(this.table, columns, this.keyColumn, this.spec.immutableColumns || []) + await this.driver.run(sql, columns.map(c => record[c])) + return key + } + + async removeItem(key: string): Promise { + await this.ensureInitialized() + await this.driver.run(`DELETE FROM ${this.q(this.table)} WHERE ${this.q(this.keyColumn)} = ?`, [key]) + } + + async listItems(): Promise { + await this.ensureInitialized() + const rows = await this.driver.all(`SELECT * FROM ${this.q(this.table)}${this.orderClause()}`, []) + return rows.map(row => this.spec.fromRecord(row)).filter((item): item is T => Boolean(item)) + } + + /** 能下推的条件走 SQL，其余在内存里过滤。 */ + async listItemsByEqFilter(filter: Record): Promise { + await this.ensureInitialized() + if (!filter || Object.keys(filter).length === 0) return this.listItems() + + const clauses: string[] = [] + const params: unknown[] = [] + const residual: Record = {} + + for (const [field, value] of Object.entries(filter)) { + if (this.isFilterable(field)) { + clauses.push(`${this.q(field)} = ?`) + params.push(this.normalize(field, value)) + } else { + residual[field] = value + } + } + + const where = clauses.length ? ` WHERE ${clauses.join(' AND ')}` : '' + const rows = await this.driver.all(`SELECT * FROM ${this.q(this.table)}${where}${this.orderClause()}`, params) + const items = rows.map(row => this.spec.fromRecord(row)).filter((item): item is T => Boolean(item)) + + const entries = Object.entries(residual) + if (entries.length === 0) return items + return items.filter(item => + entries.every(([key, value]) => (item as Record)[key] === value), + ) + } + + async listItemsByInQuery(query: { field: string, values: unknown[] }[]): Promise { + await this.ensureInitialized() + if (!Array.isArray(query) || query.length === 0) return this.listItems() + // 任何一组候选值为空，交集必然为空 + if (query.some(({ values }) => Array.isArray(values) && values.length === 0)) return [] + + const clauses: string[] = [] + const params: unknown[] = [] + const residual: { field: string, values: unknown[] }[] = [] + + for (const { field, values } of query) { + if (!Array.isArray(values)) continue + if (this.isFilterable(field)) { + clauses.push(`${this.q(field)} IN (${values.map(() => '?').join(', ')})`) + params.push(...values.map(value => this.normalize(field, value))) + } else { + residual.push({ field, values }) + } + } + + const where = clauses.length ? ` WHERE ${clauses.join(' AND ')}` : '' + const rows = await this.driver.all(`SELECT * FROM ${this.q(this.table)}${where}${this.orderClause()}`, params) + let items = rows.map(row => this.spec.fromRecord(row)).filter((item): item is T => Boolean(item)) + for (const { field, values } of residual) { + items = items.filter(item => values.includes((item as Record)[field])) + } + return items + } + + async clear(): Promise { + await this.ensureInitialized() + await this.driver.run(`DELETE FROM ${this.q(this.table)}`, []) + } + + /** + * 故意什么都不做。 + * + * 一个 driver 是被多个 storage 共用的（Postgres 下更是共用同一个连接池）， + * 任何一个 storage 关掉它，其余的就全断了。连接的生命周期归 DriverRegistry + * 管，进程退出时统一关闭。 + */ + async close(): Promise {} + + protected isFilterable(field: string): boolean { + return (this.spec.filterable || []).includes(field) + } + + /** 过滤值归一：数值列用 Number，布尔列用 0/1，与建表时的存储表示对齐。 */ + protected normalize(field: string, value: unknown): unknown { + if ((this.spec.numeric || []).includes(field)) return Number(value) + if ((this.spec.boolean || []).includes(field)) return value ? 1 : 0 + return value + } +} + +/** JSON 列的读写。存坏了就当空值，不要让一条脏数据把整个列表查询打挂。 */ +export function parseJson(value: unknown, fallback: T): T { + if (value === null || value === undefined || value === '') return fallback + if (typeof value === 'object') return value as T + try { + return JSON.parse(String(value)) as T + } catch { + return fallback + } +} + +export function stringifyJson(value: unknown): string | null { + if (value === null || value === undefined) return null + return JSON.stringify(value) +} diff --git a/src/storage/storage.test.ts b/src/storage/storage.test.ts new file mode 100644 index 0000000..5d1a0f4 --- /dev/null +++ b/src/storage/storage.test.ts @@ -0,0 +1,684 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { toDollarPlaceholders, countPlaceholders } from './driver/placeholders' +import { createDialect } from './driver/dialect' +import { SqliteDriver } from './driver/sqlite' +import { SqlKvStorage, TableSpec, parseJson, stringifyJson } from './sql_storage' +import { SqlUserStateStorage } from './sql/user_state_storage' +import { SqlOperationLogStorage } from './sql/operation_log_storage' +import { SqlHistoryManager } from './sql/history_manager' +import { HistoryMessage } from '../types/models' +import { OperationLog } from '../types/operation-log' + +function tempDir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) +} + +function tempDbPath(): string { + return path.join(tempDir('chaite-sql-'), 'test.db') +} + +async function makeDriver(): Promise { + const driver = new SqliteDriver(tempDbPath()) + await driver.ready() + return driver +} + +describe('placeholders', () => { + test('numbers placeholders in order', () => { + expect(toDollarPlaceholders('SELECT * FROM t WHERE a = ? AND b = ?')).toBe( + 'SELECT * FROM t WHERE a = $1 AND b = $2' + ) + }) + + test('leaves placeholders inside string literals alone', () => { + expect(toDollarPlaceholders('SELECT * FROM t WHERE a = ? AND b = \'why?\'')).toBe( + 'SELECT * FROM t WHERE a = $1 AND b = \'why?\'' + ) + // '' 是转义的单引号，不能被当成字符串结束 + expect(toDollarPlaceholders('SELECT \'it\'\'s a ?\' , ?')).toBe('SELECT \'it\'\'s a ?\' , $1') + }) + + test('leaves quoted identifiers and comments alone', () => { + expect(toDollarPlaceholders('SELECT "we?rd" FROM t WHERE a = ? -- trailing ?\n')).toBe( + 'SELECT "we?rd" FROM t WHERE a = $1 -- trailing ?\n' + ) + expect(toDollarPlaceholders('SELECT /* ? */ a FROM t WHERE b = ?')).toBe('SELECT /* ? */ a FROM t WHERE b = $1') + }) + + test('counts placeholders', () => { + expect(countPlaceholders('SELECT ?, ?, ?')).toBe(3) + expect(countPlaceholders('SELECT \'no ? here\'')).toBe(0) + }) +}) + +describe('dialect', () => { + test('upsert updates every column except the conflict key', () => { + const sql = createDialect('postgres').upsert('channels', ['id', 'name', 'weight'], 'id') + expect(sql).toMatch(/ON CONFLICT\("id"\) DO UPDATE SET/) + expect(sql).toMatch(/"name" = EXCLUDED\."name"/) + expect(sql).not.toMatch(/"id" = EXCLUDED\."id"/) + }) + + test('immutable columns are not overwritten on conflict', () => { + const sql = createDialect('sqlite').upsert('user_states', ['id', 'userId', 'nickname'], 'userId', ['id']) + expect(sql).toMatch(/"nickname" = EXCLUDED\."nickname"/) + expect(sql).not.toMatch(/"id" = EXCLUDED\."id"/) + }) + + test('a single-column table degrades to DO NOTHING', () => { + expect(createDialect('sqlite').upsert('t', ['id'], 'id')).toMatch(/DO NOTHING/) + }) + + test('both dialects emit the same upsert shape', () => { + const columns = ['id', 'name'] + expect(createDialect('sqlite').upsert('t', columns, 'id')).toBe(createDialect('postgres').upsert('t', columns, 'id')) + }) + + test('autoincrement differs per dialect', () => { + expect(createDialect('sqlite').autoIncrementPk()).toMatch(/AUTOINCREMENT/) + expect(createDialect('postgres').autoIncrementPk()).toMatch(/BIGSERIAL/) + }) + + test('per-column index ordering is preserved', () => { + const sql = createDialect('sqlite').createIndex('operation_logs', ['userId', { column: 'timestamp', order: 'DESC' }]) + expect(sql).toMatch(/\("userId", "timestamp" DESC\)/) + }) + + test('unsafe identifiers are rejected', () => { + expect(() => createDialect('sqlite').quoteId('a"; DROP TABLE t; --')).toThrow(/unsafe identifier/) + }) +}) + +describe('json helpers', () => { + test('tolerate garbage', () => { + expect(parseJson('{"a":1}', null)).toEqual({ a: 1 }) + expect(parseJson('not json', null)).toBeNull() + expect(parseJson('not json', {})).toEqual({}) + expect(stringifyJson(undefined)).toBeNull() + }) +}) + +interface Widget { + id?: string + name?: string + kind?: string + weight?: number + enabled?: boolean + payload?: unknown +} + +/** 一份最小 spec，覆盖提升列、数值列、布尔列和 JSON 列四种形态 */ +const widgetSpec: TableSpec = { + table: 'widgets', + columns: { + id: { type: 'text', pk: true }, + name: { type: 'text', notNull: true }, + kind: { type: 'text' }, + weight: { type: 'int', default: 1 }, + enabled: { type: 'bool', default: 0 }, + payload: { type: 'json' }, + }, + indexes: [{ columns: ['kind'] }], + filterable: ['id', 'name', 'kind', 'weight', 'enabled'], + numeric: ['weight'], + boolean: ['enabled'], + toRecord: (entity, id) => ({ + id, + name: entity.name, + kind: entity.kind, + weight: entity.weight ?? 1, + enabled: entity.enabled ? 1 : 0, + payload: stringifyJson(entity.payload), + }), + fromRecord: record => ({ + id: record.id as string, + name: record.name as string, + kind: record.kind as string, + weight: Number(record.weight), + enabled: Boolean(record.enabled), + payload: parseJson(record.payload, {}), + }), +} + +describe('SqlKvStorage', () => { + async function makeStorage() { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + return storage + } + + test('round-trips an item', async () => { + const storage = await makeStorage() + expect(await storage.setItem('w1', { name: 'first', kind: 'a', weight: 5, enabled: true, payload: { deep: [1, 2] } })).toBe('w1') + + const loaded = await storage.getItem('w1') + expect(loaded).toMatchObject({ name: 'first', weight: 5, enabled: true, payload: { deep: [1, 2] } }) + expect(await storage.getItem('missing')).toBeNull() + await storage.driver.close() + }) + + test('setItem upserts rather than duplicating', async () => { + const storage = await makeStorage() + await storage.setItem('w1', { name: 'first', kind: 'a' }) + await storage.setItem('w1', { name: 'renamed', kind: 'b' }) + + const all = await storage.listItems() + expect(all).toHaveLength(1) + expect(all[0]).toMatchObject({ name: 'renamed', kind: 'b' }) + await storage.driver.close() + }) + + test('generates an id when none is given', async () => { + const storage = await makeStorage() + const id = await storage.setItem('', { name: 'anon' }) + expect(id).toBeTruthy() + expect((await storage.getItem(id))?.name).toBe('anon') + await storage.driver.close() + }) + + test('eq filter pushes known columns down and filters the rest in memory', async () => { + const storage = await makeStorage() + await storage.setItem('w1', { name: 'a', kind: 'x', weight: 1 }) + await storage.setItem('w2', { name: 'b', kind: 'x', weight: 2 }) + await storage.setItem('w3', { name: 'c', kind: 'y', weight: 1 }) + + expect((await storage.listItemsByEqFilter({ kind: 'x' })).map(i => i.id).sort()).toEqual(['w1', 'w2']) + // weight 是数值列：传字符串也应该匹配得上 + expect((await storage.listItemsByEqFilter({ weight: '1' })).map(i => i.id).sort()).toEqual(['w1', 'w3']) + // 不在 filterable 里的字段只能在内存里比 + expect(await storage.listItemsByEqFilter({ kind: 'x', nope: 'zzz' })).toHaveLength(0) + expect(await storage.listItemsByEqFilter({})).toHaveLength(3) + await storage.driver.close() + }) + + test('boolean columns normalise to 0/1 in the query', async () => { + const storage = await makeStorage() + await storage.setItem('on', { name: 'on', enabled: true }) + await storage.setItem('off', { name: 'off', enabled: false }) + + expect((await storage.listItemsByEqFilter({ enabled: true })).map(i => i.id)).toEqual(['on']) + expect((await storage.listItemsByEqFilter({ enabled: false })).map(i => i.id)).toEqual(['off']) + await storage.driver.close() + }) + + test('in-query intersects and short-circuits on an empty set', async () => { + const storage = await makeStorage() + await storage.setItem('w1', { name: 'a', kind: 'x' }) + await storage.setItem('w2', { name: 'b', kind: 'y' }) + await storage.setItem('w3', { name: 'c', kind: 'x' }) + + const hits = await storage.listItemsByInQuery([ + { field: 'kind', values: ['x'] }, + { field: 'name', values: ['a', 'c'] }, + ]) + expect(hits.map(i => i.id).sort()).toEqual(['w1', 'w3']) + expect(await storage.listItemsByInQuery([{ field: 'kind', values: [] }])).toEqual([]) + expect(await storage.listItemsByInQuery([])).toHaveLength(3) + await storage.driver.close() + }) + + test('removeItem and clear', async () => { + const storage = await makeStorage() + await storage.setItem('w1', { name: 'a' }) + await storage.setItem('w2', { name: 'b' }) + + await storage.removeItem('w1') + expect(await storage.listItems()).toHaveLength(1) + await storage.clear() + expect(await storage.listItems()).toHaveLength(0) + await storage.driver.close() + }) + + test('initialize is idempotent under a race', async () => { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await Promise.all([storage.initialize(), storage.initialize(), storage.initialize()]) + await storage.setItem('w1', { name: 'a' }) + expect(await storage.listItems()).toHaveLength(1) + await driver.close() + }) + + test('close() does not disconnect the shared driver', async () => { + const driver = await makeDriver() + const a = new SqlKvStorage(driver, widgetSpec) + await a.initialize() + // 八个 storage 共用一个 driver，任何一个 close 都不该把连接关掉 + await a.close() + await a.setItem('w1', { name: 'still works' }) + expect(await a.listItems()).toHaveLength(1) + await driver.close() + }) +}) + +/* + * 注意：Channel / ChatPreset / ToolDTO / TriggerDTO 这四个具体 spec 的测试没有放在 + * 这里。它们要 import 对应的 DTO 类，而那条链路会走到 types 桶文件 -> core -> + * adapters，chaite 现有的模块循环在 jest 的 CJS 环境下会炸（ESM 打包时靠提升侥幸 + * 不炸）。把 jest 切到 ESM 或者拆掉这些环都是独立的工程。 + * + * 这四个 spec 的序列化行为由宿主插件里的 parity 测试覆盖：那边用真实数据文件比对 + * 新旧两套实现的读写结果，正好是最该保证的东西。 + */ + +describe('SqlUserStateStorage', () => { + test('keys on userId and keeps the surrogate id stable', async () => { + const driver = await makeDriver() + const storage = new SqlUserStateStorage(driver) + await storage.initialize() + + await storage.setItem('10001', { userId: '10001', nickname: 'first', conversations: [], settings: {}, current: {} } as never) + const firstRow = await driver.get<{ id: string }>('SELECT id FROM user_states', []) + + await storage.setItem('10001', { userId: '10001', nickname: 'second', conversations: [], settings: {}, current: {} } as never) + const rows = await driver.all<{ id: string, nickname: string }>('SELECT * FROM user_states', []) + + expect(rows).toHaveLength(1) + expect(rows[0].nickname).toBe('second') + expect(rows[0].id).toBe(firstRow?.id) + await driver.close() + }) + + test('collapses duplicate userIds before adding the unique index', async () => { + const driver = await makeDriver() + // 先建出老结构（没有唯一索引），再插两行同 userId，模拟并发写留下的重复 + await driver.exec(`CREATE TABLE IF NOT EXISTS user_states ( + id TEXT PRIMARY KEY, userId TEXT NOT NULL, nickname TEXT, card TEXT, + conversations TEXT NOT NULL, settings TEXT NOT NULL, current TEXT NOT NULL, updatedAt INTEGER + )`) + const insert = `INSERT INTO user_states (id, userId, nickname, card, conversations, settings, current, updatedAt) + VALUES (?, ?, ?, NULL, '[]', '{}', '{}', ?)` + await driver.run(insert, ['uuid-old', '555', 'stale', 1000]) + await driver.run(insert, ['uuid-new', '555', 'fresh', 2000]) + + const storage = new SqlUserStateStorage(driver) + await storage.initialize() + + const rows = await driver.all<{ id: string, nickname: string }>('SELECT * FROM user_states', []) + expect(rows).toHaveLength(1) + expect(rows[0].nickname).toBe('fresh') + expect(rows[0].id).toBe('uuid-new') + await driver.close() + }) +}) + +describe('SqlOperationLogStorage', () => { + function log(id: string, timestamp: number): OperationLog { + return { id, timestamp, type: 'llm.call', level: 'info', summary: `log ${id}` } + } + + test('prunes down to maxEntries, oldest first', async () => { + const driver = await makeDriver() + const storage = new SqlOperationLogStorage(driver, 10) + await storage.initialize() + + for (let i = 0; i < 30; i++) { + await storage.setItem(`log-${String(i).padStart(3, '0')}`, log(`log-${String(i).padStart(3, '0')}`, i)) + } + // 直接调一次强制裁剪，避免依赖写入计数的触发时机 + await storage.setMaxEntries(10) + + const rows = await driver.all<{ id: string }>('SELECT id FROM operation_logs ORDER BY timestamp ASC', []) + expect(rows).toHaveLength(10) + // 保留的应该是最新的十条 + expect(rows[0].id).toBe('log-020') + await driver.close() + }) + + test('maxEntries of 0 disables pruning', async () => { + const driver = await makeDriver() + const storage = new SqlOperationLogStorage(driver, 0) + await storage.initialize() + + for (let i = 0; i < 20; i++) await storage.setItem(`log-${i}`, log(`log-${i}`, i)) + await storage.setMaxEntries(0) + + expect(await driver.all('SELECT id FROM operation_logs', [])).toHaveLength(20) + await driver.close() + }) + + test('reads back the full payload and orders newest first', async () => { + const driver = await makeDriver() + const storage = new SqlOperationLogStorage(driver, 0) + await storage.initialize() + + await storage.setItem('a', { ...log('a', 1), userId: 'u1' }) + await storage.setItem('b', { ...log('b', 2), userId: 'u2' }) + + expect((await storage.listItems()).map(l => l.id)).toEqual(['b', 'a']) + expect((await storage.listItemsByEqFilter({ userId: 'u1' })).map(l => l.id)).toEqual(['a']) + await driver.close() + }) +}) + +describe('SqlHistoryManager', () => { + async function makeHistory() { + const driver = await makeDriver() + const manager = new SqlHistoryManager(driver, tempDir('chaite-img-')) + await manager.initialize() + return manager + } + + function message(id: string, parentId: string | null, text: string): HistoryMessage { + return { id, parentId, role: 'user', content: [{ type: 'text', text }] } as unknown as HistoryMessage + } + + test('walks the parent chain back to the root, oldest first', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('m1', null, 'first'), 'c1') + await manager.saveHistory(message('m2', 'm1', 'second'), 'c1') + await manager.saveHistory(message('m3', 'm2', 'third'), 'c1') + + const chain = await manager.getHistory('m3') + expect(chain.map(m => m.id)).toEqual(['m1', 'm2', 'm3']) + await manager.driver.close() + }) + + test('a parentId cycle truncates instead of hanging', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('a', 'b', 'one'), 'c1') + await manager.saveHistory(message('b', 'a', 'two'), 'c1') + + // 改造前这里是无限递归；带访问集合后应该有限步停下 + const chain = await manager.getHistory('a') + expect(chain.map(m => m.id).sort()).toEqual(['a', 'b']) + await manager.driver.close() + }) + + test('saveHistories writes every message in one transaction', async () => { + const manager = await makeHistory() + await manager.saveHistories([message('m1', null, 'a'), message('m2', 'm1', 'b')], 'c1') + expect(await manager.getHistory(undefined, 'c1')).toHaveLength(2) + await manager.driver.close() + }) + + test('removeHistory reparents children instead of breaking the chain', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('m1', null, 'first'), 'c1') + await manager.saveHistory(message('m2', 'm1', 'second'), 'c1') + await manager.saveHistory(message('m3', 'm2', 'third'), 'c1') + + await manager.removeHistory('m2', 'c1') + + const chain = await manager.getHistory('m3') + expect(chain.map(m => m.id)).toEqual(['m1', 'm3']) + await manager.driver.close() + }) + + test('base64 images are written to disk and restored on read', async () => { + const manager = await makeHistory() + const png = Buffer.from('hello world, pretend this is a png').toString('base64') + await manager.saveHistory( + { id: 'm1', parentId: null, role: 'user', content: [{ type: 'image', image: png }] } as unknown as HistoryMessage, + 'c1' + ) + + // 行里存的应该是引用而不是图片本体 + const row = await manager.driver.get<{ messageData: string }>('SELECT messageData FROM history WHERE id = ?', ['m1']) + expect(row?.messageData).toMatch(/\$image:[a-f0-9]+:\.jpg/) + expect(row?.messageData).not.toContain(png) + expect(fs.readdirSync(manager.imagesDir)).toHaveLength(1) + + const restored = await manager.getOneHistory('m1', 'c1') + expect((restored?.content as { image?: string }[])[0].image).toBe(png) + await manager.driver.close() + }) + + test('a missing image file degrades to a [图片] placeholder', async () => { + const manager = await makeHistory() + const png = Buffer.from('another fake png').toString('base64') + await manager.saveHistory( + { id: 'm1', parentId: null, role: 'user', content: [{ type: 'image', image: png }] } as unknown as HistoryMessage, + 'c1' + ) + for (const file of fs.readdirSync(manager.imagesDir)) fs.unlinkSync(path.join(manager.imagesDir, file)) + + const restored = await manager.getOneHistory('m1', 'c1') + expect(restored?.content as unknown).toEqual([{ type: 'text', text: '[图片]' }]) + await manager.driver.close() + }) + + test('cleanupUnusedImages keeps referenced files only', async () => { + const manager = await makeHistory() + const png = Buffer.from('referenced').toString('base64') + await manager.saveHistory( + { id: 'm1', parentId: null, role: 'user', content: [{ type: 'image', image: png }] } as unknown as HistoryMessage, + 'c1' + ) + fs.writeFileSync(path.join(manager.imagesDir, 'orphan.png'), 'nobody references me') + + const result = await manager.cleanupUnusedImages() + expect(result.deleted).toBe(1) + expect(fs.readdirSync(manager.imagesDir)).toHaveLength(1) + await manager.driver.close() + }) + + test('prune deletes only what predates the cutoff', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('m1', null, 'old'), 'c1') + await manager.saveHistory(message('m2', null, 'new'), 'c1') + // createdAt 是写入时生成的，这里直接改成一个确定的旧值 + await manager.driver.run('UPDATE history SET createdAt = ? WHERE id = ?', ['2000-01-01T00:00:00.000Z', 'm1']) + + const cutoff = '2001-01-01T00:00:00.000Z' + expect(await manager.countHistoryBefore({ before: cutoff })).toBe(1) + + const result = await manager.pruneHistory({ before: cutoff }) + expect(result.deleted).toBe(1) + expect(result.truncated).toBe(false) + expect((await manager.getHistory(undefined, 'c1')).map(m => m.id)).toEqual(['m2']) + await manager.driver.close() + }) + + test('prune can be scoped to a conversation prefix', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('m1', null, 'a'), 'group:111') + await manager.saveHistory(message('m2', null, 'b'), 'private:222') + await manager.driver.run('UPDATE history SET createdAt = ?', ['2000-01-01T00:00:00.000Z']) + + const result = await manager.pruneHistory({ before: '2001-01-01T00:00:00.000Z', conversationPrefix: 'group:' }) + expect(result.deleted).toBe(1) + expect((await manager.getHistory(undefined, 'private:222'))).toHaveLength(1) + await manager.driver.close() + }) + + test('deleteConversation removes the whole conversation', async () => { + const manager = await makeHistory() + await manager.saveHistory(message('m1', null, 'a'), 'c1') + await manager.saveHistory(message('m2', null, 'b'), 'c2') + + await manager.deleteConversation('c1') + expect(await manager.getHistory(undefined, 'c1')).toHaveLength(0) + expect(await manager.getHistory(undefined, 'c2')).toHaveLength(1) + await manager.driver.close() + }) +}) + +describe('lifecycle and concurrency', () => { + test('a racing ensureInitialized never sees a half-built user_states', async () => { + const driver = await makeDriver() + const storage = new SqlUserStateStorage(driver) + + // 三个并发调用：只要有任何一个在唯一索引建好之前就放行，后面的写入就可能 + // 插进重复 userId + await Promise.all([ + storage.initialize(), + storage.ensureInitialized(), + storage.ensureInitialized(), + ]) + + const indexes = await driver.all<{ name: string }>( + 'SELECT name FROM sqlite_master WHERE type = ? AND tbl_name = ?', ['index', 'user_states'] + ) + expect(indexes.map(i => i.name)).toContain('uniq_user_states_userId') + await driver.close() + }) + + test('a failed afterInitialize leaves the storage retryable', async () => { + const driver = await makeDriver() + let attempts = 0 + + class FlakyStorage extends SqlKvStorage { + protected async afterInitialize(): Promise { + attempts++ + if (attempts === 1) throw new Error('afterInitialize boom') + } + } + const storage = new FlakyStorage(driver, widgetSpec) + + await expect(storage.initialize()).rejects.toThrow('afterInitialize boom') + // 关键在于没有被标成已就绪：下一次调用要重跑整条链路，而不是带着没建好的 + // 模式继续用下去 + expect(attempts).toBe(1) + + // 后续任何一次访问都会触发重试，这次能成 + await storage.setItem('w1', { name: 'ok' }) + expect((await storage.getItem('w1'))?.name).toBe('ok') + expect(attempts).toBe(2) + await driver.close() + }) + + test('close() rejects new work instead of queueing onto a dying connection', async () => { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + await storage.setItem('w1', { name: 'a' }) + + await driver.close() + + await expect(driver.run('INSERT INTO widgets (id,name) VALUES (?,?)', ['w2', 'b'])).rejects.toThrow(/closing|closed/) + await expect(driver.get('SELECT * FROM widgets', [])).rejects.toThrow(/closing|closed/) + }) + + test('concurrent close() calls share one shutdown', async () => { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + // 三个并发 close 不该各关一遍（第二次 close 原生连接会报错） + await Promise.all([driver.close(), driver.close(), driver.close()]) + await expect(driver.get('SELECT 1', [])).rejects.toThrow(/closing|closed/) + }) + + test('in-flight writes finish before close() tears the connection down', async () => { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + // driver.run 是同步入队的，所以这三条在 close 发起前就已经在队列里了。 + // 经由 storage.setItem 会先 await ensureInitialized，那时 close 已经开始， + // 属于「关闭后才到达」，应当被拒绝——那是另一个用例。 + const writes = [ + driver.run('INSERT INTO widgets (id,name) VALUES (?,?)', ['w1', 'a']), + driver.run('INSERT INTO widgets (id,name) VALUES (?,?)', ['w2', 'b']), + driver.run('INSERT INTO widgets (id,name) VALUES (?,?)', ['w3', 'c']), + ] + const closing = driver.close() + await Promise.all(writes) + await closing + + // 重新打开，确认三条都落盘了 + const reopened = new SqliteDriver(driver.dbPath) + await reopened.ready() + expect(await reopened.all('SELECT id FROM widgets', [])).toHaveLength(3) + await reopened.close() + }) +}) + +describe('DriverRegistry', () => { + test('concurrent init() opens exactly one set of drivers', async () => { + const { DriverRegistry } = await import('./driver/registry') + const registry = new DriverRegistry() + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'reg-')) + + await Promise.all([ + registry.init({ dialect: 'sqlite', dataDir }), + registry.init({ dialect: 'sqlite', dataDir }), + registry.init({ dialect: 'sqlite', dataDir }), + ]) + + // 三个逻辑库、三个文件，绝不能因为并发多开一套 + expect(registry.listUnique()).toHaveLength(3) + await registry.close() + }) + + test('a failed init() leaves nothing open and stays retryable', async () => { + const { DriverRegistry } = await import('./driver/registry') + const registry = new DriverRegistry() + + await expect( + registry.init({ dialect: 'mysql' } as never) + ).rejects.toThrow(/不支持的数据库方言/) + + expect(registry.isInitialized()).toBe(false) + expect(registry.listUnique()).toHaveLength(0) + + // 修好之后应该还能正常初始化 + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'reg-retry-')) + await registry.init({ dialect: 'sqlite', dataDir }) + expect(registry.isInitialized()).toBe(true) + await registry.close() + }) + + test('close() is idempotent', async () => { + const { DriverRegistry } = await import('./driver/registry') + const registry = new DriverRegistry() + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'reg-close-')) + await registry.init({ dialect: 'sqlite', dataDir }) + + await Promise.all([registry.close(), registry.close()]) + expect(registry.isInitialized()).toBe(false) + }) +}) + +describe('column reconciliation on upgrade', () => { + test('adds columns that an older table is missing', async () => { + const driver = await makeDriver() + // 造一张「老版本」表：只有主键和 name，其余列都还不存在 + await driver.exec('CREATE TABLE widgets (id TEXT PRIMARY KEY, name TEXT NOT NULL)') + await driver.run('INSERT INTO widgets (id, name) VALUES (?, ?)', ['old', 'legacy row']) + + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + const columns = await driver.all<{ name: string }>('PRAGMA table_info("widgets")', []) + expect(columns.map(c => c.name).sort()).toEqual(['enabled', 'id', 'kind', 'name', 'payload', 'weight']) + + // 老行还在，而且现在能按当前 schema 正常写入——补列之前这里会报 + // "table widgets has no column named kind" + await storage.setItem('new', { name: 'fresh', kind: 'a', weight: 3, enabled: true }) + expect(await storage.listItems()).toHaveLength(2) + expect((await storage.getItem('old'))?.name).toBe('legacy row') + expect((await storage.getItem('new'))?.kind).toBe('a') + await driver.close() + }) + + test('a defaulted column keeps its default for pre-existing rows', async () => { + const driver = await makeDriver() + await driver.exec('CREATE TABLE widgets (id TEXT PRIMARY KEY, name TEXT NOT NULL)') + await driver.run('INSERT INTO widgets (id, name) VALUES (?, ?)', ['old', 'legacy']) + + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + + // weight 的 spec 默认值是 1，老行补列后应当拿到它 + const row = await driver.get<{ weight: number }>('SELECT weight FROM widgets WHERE id = ?', ['old']) + expect(Number(row?.weight)).toBe(1) + await driver.close() + }) + + test('reconciliation is a no-op on an already current table', async () => { + const driver = await makeDriver() + const storage = new SqlKvStorage(driver, widgetSpec) + await storage.initialize() + await storage.setItem('w1', { name: 'a', kind: 'x' }) + + // 第二个实例对着同一张表初始化，不该动任何东西 + const again = new SqlKvStorage(driver, widgetSpec) + await again.initialize() + expect(await again.listItems()).toHaveLength(1) + await driver.close() + }) +}) diff --git a/src/types/common.ts b/src/types/common.ts index 7613e87..ca33429 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -4,7 +4,9 @@ import { HistoryManager, SendMessageOption } from './adapter' import { PostProcessor, PreProcessor } from './processors' import { DeSerializable, Serializable, Wait } from './cloud' import { EventMessage } from './external' -import { AbstractClient } from '../adapters' +// 只作类型使用。写成值导入会形成 types -> adapters -> types 的运行时环， +// 并把 PdfParser 之类用了 import.meta 的模块一并拖进任何 import 了 types 的地方。 +import type { AbstractClient } from '../adapters' import type { Chaite } from '../core' import type { ToolExecutor } from '../agent/contracts' diff --git a/src/types/tools.ts b/src/types/tools.ts index f957580..bada6f3 100644 --- a/src/types/tools.ts +++ b/src/types/tools.ts @@ -91,6 +91,11 @@ export class ToolsGroupDTO extends AbstractShareable implements T constructor(params: Partial) { super(params) this.modelType = 'settings' + // AbstractShareable 只搬它自己认识的字段，这两个是 ToolsGroupDTO 独有的。 + // 漏掉的话 new ToolsGroupDTO({ toolIds }) 出来的对象 toolIds 永远是 undefined， + // 工具组等于形同虚设。 + this.toolIds = params.toolIds || [] + this.isDefault = params.isDefault } toolIds: string[] diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index 4f38db2..edad3d0 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,9 +1,12 @@ +// 从 types/common 直接取，不走 types 桶文件：桶文件会把 trigger（依赖只发 ESM +// 的 node-schedule）和 adapters（PdfParser 里用了 import.meta）一并拉进来， +// 任何 import 到 helpers 的模块都会被连累。 import { ChaiteContext, DefaultLogger, ILogger, MultipleKeyStrategy, MultipleKeyStrategyChoice, -} from '../types' +} from '../types/common' import { AsyncLocalStorage } from 'async_hooks' import fs from 'fs/promises' import path from 'path' diff --git a/src/utils/logger.ts b/src/utils/logger.ts index b3e2b4f..bb99eeb 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -1,5 +1,5 @@ -import { DefaultLogger, ILogger } from '../types' -import { asyncLocalStorage } from './index' +import { DefaultLogger, ILogger } from '../types/common' +import { asyncLocalStorage } from './helpers' export function getLogger (): ILogger { const chaite = asyncLocalStorage.getStore()?.chaite diff --git a/test/stubs/dirname.cjs b/test/stubs/dirname.cjs new file mode 100644 index 0000000..bf8c490 --- /dev/null +++ b/test/stubs/dirname.cjs @@ -0,0 +1,8 @@ +/** + * 测试用的 src/__dirname.ts 替身。 + * + * 原文件用 import.meta.url 推导 __dirname，而 jest 跑在 CJS 下、 + * ts-jest 以 module=commonjs 编译，import.meta 在这种组合下无法编译 + * （TS1343）。CJS 里本来就有 __dirname，直接用即可。 + */ +module.exports = { __dirname: __dirname } diff --git a/test/stubs/node-schedule.cjs b/test/stubs/node-schedule.cjs new file mode 100644 index 0000000..d25feae --- /dev/null +++ b/test/stubs/node-schedule.cjs @@ -0,0 +1,33 @@ +/** + * 测试用的 @karinjs/node-schedule 替身。 + * + * 那个包只发 ESM（dist/index.mjs），而 jest 在 CJS 模式下无法加载 .mjs—— + * 不管 transform 怎么配都一样，因为 jest 是按扩展名决定模块系统的。 + * 真要修得用 --experimental-vm-modules 把整个测试套件切到 ESM，那是另一件事。 + * + * types/trigger.ts 在模块顶层 import 了它，于是任何 import 到 types 桶文件的 + * 测试都会被牵连——包括完全用不到定时任务的存储层测试。这里给个最小替身， + * 真正要测调度行为时再换成完整实现。 + */ +class Job { + constructor (name) { + this.name = name + } + + cancel () { + return true + } +} + +module.exports = { + __esModule: true, + default: { + scheduleJob: () => new Job('stub'), + cancelJob: () => true, + gracefulShutdown: async () => {}, + }, + Job, + scheduleJob: () => new Job('stub'), + cancelJob: () => true, + gracefulShutdown: async () => {}, +}