From 20371e42b4b3deabe96bac2b4a6434286d8f5907 Mon Sep 17 00:00:00 2001 From: moeSegFault Date: Sun, 27 Sep 2026 16:22:13 +0800 Subject: [PATCH 1/2] feat(workspace): add shared training and merge workflows Train CPU, CUDA, and iGPU on bounded multi-directory corpora using the serving model, with end-to-end telemetry and opt-in summaries. Merge workspaces through provenance-aware model contributions, absolute digest cache, telemetry run IDs, scoped ignore rules, and guarded source cleanup. Preserve legacy train options and scan result scope. Verified with the 27-test Windows Debug suite and physical CUDA/iGPU training smoke; cross-platform Actions validation follows in the release PR. --- CMakeLists.txt | 7 +- README.md | 33 +- docs/contextual-learning.md | 10 +- docs/merge-design.md | 94 +++ ...online-model-parameter-audit-2026-09-27.md | 58 ++ docs/summary-output-design.md | 11 + docs/telemetry.md | 10 + docs/train-mode-validation.md | 77 ++ include/same/application.hpp | 28 + include/same/merge.hpp | 29 + include/same/model_store.hpp | 27 + include/same/resources.hpp | 40 +- include/same/store.hpp | 15 + include/same/workspace.hpp | 21 + src/application.cpp | 697 +++++++++++++++++- src/main.cpp | 212 +++++- src/merge.cpp | 556 ++++++++++++++ src/model_store.cpp | 532 ++++++++++++- src/resources.cpp | 44 +- src/store.cpp | 102 ++- src/workspace.cpp | 97 ++- tests/igpu_routing_tests.cpp | 32 + tests/integration_tests.cpp | 25 +- tests/merge_tests.cpp | 370 ++++++++++ tests/model_store_tests.cpp | 18 +- tests/training_tests.cpp | 201 +++++ 26 files changed, 3227 insertions(+), 119 deletions(-) create mode 100644 docs/merge-design.md create mode 100644 docs/research/online-model-parameter-audit-2026-09-27.md create mode 100644 docs/train-mode-validation.md create mode 100644 include/same/merge.hpp create mode 100644 src/merge.cpp create mode 100644 tests/merge_tests.cpp create mode 100644 tests/training_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8197879..6aa81af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ include(cmake/Dependencies.cmake) add_library(same_core src/config.cpp src/files.cpp src/store.cpp src/compute_cpu.cpp src/resources.cpp src/application.cpp src/run_lock.cpp src/terminal.cpp src/walk.cpp src/dispatch.cpp src/online_model.cpp src/model_store.cpp src/telemetry.cpp src/workspace.cpp) +target_sources(same_core PRIVATE src/merge.cpp) # Propagate the public C++ contract without changing CUDA's separate standard. # 传播公开 C++ 契约,不改变 CUDA 独立语言标准;头文件集提供 IDE 元数据。 target_compile_features(same_core PUBLIC cxx_std_23) @@ -37,6 +38,7 @@ target_sources(same_core PUBLIC FILE_SET HEADERS BASE_DIRS include FILES include/same/store.hpp include/same/terminal.hpp include/same/detail/windows_metadata.hpp include/same/detail/windows_path.hpp include/same/walk.hpp include/same/detail/completion_jobs.hpp include/same/detail/dispatch.hpp include/same/detail/online_model.hpp include/same/model_store.hpp include/same/telemetry.hpp include/same/workspace.hpp) +target_sources(same_core PUBLIC FILE_SET HEADERS FILES include/same/merge.hpp) target_link_libraries(same_core PUBLIC sqlite3 BLAKE3::blake3 tomlplusplus::tomlplusplus Threads::Threads) if(WIN32) # Include the opt-in in every executable, including tests, for CRT/STL long paths. @@ -102,12 +104,15 @@ add_executable(contextual_model_benchmark EXCLUDE_FROM_ALL tools/contextual_mode target_link_libraries(contextual_model_benchmark PRIVATE same_core) if(BUILD_TESTING) - foreach(test igpu_compute igpu_routing compute files config store resources adaptive_routing async_scan run_lock terminal walk completion hash_retry dispatch online_model contextual_model model_store online_resources unified_cuda telemetry workspace) + foreach(test igpu_compute igpu_routing compute files config store resources adaptive_routing async_scan run_lock terminal walk completion hash_retry dispatch online_model contextual_model model_store online_resources unified_cuda telemetry workspace training merge) add_executable(${test}_tests tests/${test}_tests.cpp) target_link_libraries(${test}_tests PRIVATE same_core) add_test(NAME ${test} COMMAND ${test}_tests) set_tests_properties(${test} PROPERTIES TIMEOUT 120) endforeach() + target_compile_definitions(training_tests PRIVATE SAME_TEST_ROOT="${CMAKE_CURRENT_SOURCE_DIR}") + target_compile_definitions(merge_tests PRIVATE SAME_TEST_ROOT="${CMAKE_CURRENT_SOURCE_DIR}") + target_compile_definitions(model_store_tests PRIVATE SAME_TEST_ROOT="${CMAKE_CURRENT_SOURCE_DIR}") add_test(NAME build_policy COMMAND "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/build_policy.cmake") add_executable(integration_tests tests/integration_tests.cpp) diff --git a/README.md b/README.md index 56016fd..06eae87 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,13 @@ same scan --igpu --rehash # 请求核显,仍允许安全回退 / reques same scan -r --no-pgo # 关闭运行时路由分析器 / disable runtime routing analyzer same scan -r --no-telemetry # 本轮不写遥测库 / disable telemetry persistence for this run same scan -r --summary # 显示汇总、数据库与性能统计 / show all statistics +same train # 显式训练三后端;默认读取当前目录 / opt-in three-backend training +same train --summary # 展示训练结果、阶段耗时、模型与遥测 / training report +same train ../samples ../more --summary # 多目录共用选样预算 / multiple corpora +same train --corpus=../samples # 兼容旧写法 / compatible corpus alias +same train --no-telemetry # 训练模型但不写本轮遥测库 / train without a telemetry run +same merge ../other --summary # 合并另一个工作区 / merge another workspace +same merge . -r --move --summary # 合并后代工作区后移除其 .same / move descendant state same new # 生成标准配置与推荐忽略规则 / deploy defaults and ignore same clean # 删除当前 .same / remove this directory's .same same clean -r # 也删除子目录中的 .same / remove descendant states too @@ -79,7 +86,29 @@ same --version `same` dispatches to `same scan`; implicit scan options such as `same -r --cpu` remain valid. Scans are **nonrecursive by default**; `-r` / `--recursive` enables descent. CPU overrides the backend; rehash bypasses this run's digest cache. Commands use the current directory, without a directory argument. A shallow scan prunes unseen descendant cache records from an earlier recursive scan; a later recursive scan rebuilds them rather than leaking stale results. -### 初始化与清理 / Initialize and clean +### 显式训练 / Explicit training + +`same train [dirs...] [--max-files=N] [--max-bytes=N] [--summary] [--no-telemetry]` 是独立的选择性训练命令。未指定目录时语料为当前工作目录;可列出多个目录,统一按大小分层并共用文件数/字节预算,重叠目录会被拒绝。旧 `--corpus=DIR` 仍可作为目录别名;始终递归读取普通文件,并遵守**语料目录**的 `.same/ignore`,不跟随链接。训练结果保存到**当前工作目录**的 `.same/model.db`;此后在**同一工作目录**运行的普通自动扫描直接加载并继续更新这个模型,而不是使用另一套训练专用模型。`--corpus` 只决定读取哪些训练文件,不改变模型归属;若训练时切换到临时目录、扫描时又切回原目录,扫描就不会看到该模型。训练不会写语料文件、正常扫描的 `state.db` 或重复文件结果。默认按文件大小区间有界选取最多 128 个文件、合计最多 2 GiB 的文件输入;上限可用参数调大,文件数至多 4096。零长度文件不训练;若选中语料没有达到 `gpu_min_bytes` 的文件,命令会拒绝启动设备。语料应有多个代表性的大小和真实 I/O 特性。 + +每个选中文件在自动模式的**相同缓冲预算**下由 CPU、CUDA dGPU、OpenCL iGPU 各计算一次完整摘要,并逐字节比较全部 32 字节摘要。后端顺序轮换,尽量减弱固定的冷热缓存次序偏差,但不能消除操作系统缓存与共享 I/O 混杂。训练要求三个后端都真正可用;设备缺失、失败、回退、文件变化或摘要不一致都会使本轮失败,**不把 CPU 回退记为 GPU 样本,也不保存本轮训练增量**。成功后按原有设备身份键合并到模型库;下一轮普通 `auto` 扫描可以加载匹配的历史。当前训练不会证明 GPU 在该机器上更快,也不提供统计置信区间;几乎全是小文件的语料可能不能改善默认 16 MiB 以上的路由。 + +`same train [dirs...] [--max-files=N] [--max-bytes=N] [--summary] [--no-telemetry]` explicitly trains on a bounded, size-stratified set of real files. The corpus defaults to the current directory; multiple positional directories share one global size/byte budget and overlapping roots are rejected. The old `--corpus=DIR` spelling remains an alias. Inputs are read-only, while the model remains in the current workspace. CPU, CUDA and iGPU must each finish every selected file with matching complete digests. Unavailable devices, fallback, changing inputs and mismatches fail without publishing run deltas. No scan cache or duplicate output is modified. Default limits are 128 files and 2 GiB of logical inputs, with a 4096-file maximum. Training samples include real file I/O and contention; rotating backend submission order reduces but cannot remove cache-order bias. Subsequent automatic scans reuse only matching device/configuration identities. + +训练默认在当前工作区的 `.same/telemetry.db` 写入 `command=train` 运行:成功与失败状态、选样/初始化/测量/保存/输出阶段、每个 CPU/CUDA/iGPU 尝试,以及完整的终态样本数、读字节、模型保存和启动耗时。终态指标不依赖可能丢弃的逐任务跨度;`--summary` 在 stderr 展示运行 ID、事件丢弃/错误与含遥测排空的总时间;未请求汇总时不打印这些统计。`--no-telemetry` 只关闭本轮遥测,**不关闭训练或模型持久化**。语料绝对路径进入运行配置,可能敏感;逐任务跨度不记录文件内容或路径。[遥测字段与查询](docs/telemetry.md) / Training writes a `command=train` run to `.same/telemetry.db` by default, including lifecycle status, five stage spans, per-backend attempts, and complete final counters. Per-attempt spans may be dropped; final metrics and loss counters remain separate. `--no-telemetry` disables only the diagnostic run, not training or model persistence. The corpus path in run configuration can be sensitive; task spans contain no file path or content. + +`--summary` 与扫描遵循相同约定:默认不输出完整报告;加上后在 stderr 输出 Results、Storage and I/O、Performance、Online routing model、Model persistence 与 Telemetry 分区。交互终端使用可读单位和彩色状态,重定向时默认使用机器可解析的 `key=value`;也可指定 `--format=pretty|tsv` 与 `--color=auto|always|never`。成功训练默认不输出统计或结果行,只有警告与失败仍显示。遥测的 `dropped` 指丢失逐次诊断事件,不代表训练样本未保存。 / `--summary` follows scan conventions: a sectioned, human-readable stderr report on a terminal and machine-readable key=value after redirection, with explicit format and color overrides. Successful training is quiet without `--summary`; warnings and failures remain visible. Dropped telemetry events are not dropped model samples. + +`--max-files` 是上限而不是保证:每个四倍大小区间最多保留 64 个候选,训练前仍需遍历整个语料的元数据。CUDA 启动耗时输出为各工作线程之和,不应直接加到进程墙钟时间上;核显发现与启动时间单独列出。 / `--max-files` is a ceiling, not a promised count: at most 64 candidates are retained per factor-four size band, and metadata for the entire corpus is still walked. CUDA setup output is a sum over workers, not additive to wall time; iGPU discovery and setup are reported separately. + +`--max-bytes` 限制的是**被选文件的逻辑字节总和**;三个后端各读一次,因此应用层累计读取约为该值的三倍,实际磁盘 I/O 还受系统缓存影响。训练会占用设备和 I/O,宜在空闲时对有代表性的语料运行。 / `--max-bytes` limits selected logical bytes; three backends each read them once, so application-level reads are roughly three times that limit, while physical disk I/O depends on OS caching. Use a representative corpus and run when the machine is otherwise idle. + +### 工作区合并 / Workspace merge + +`same merge [dir] [-r] [--move] [--summary]` 将源工作区的模型来源贡献、绝对路径摘要缓存和按运行 ID 去重的遥测导入当前工作区;`dir` 默认当前目录,`-r` 查找后代 `.same`(自身不合并自身)。目的工作区的 `.same/config.toml` 永不覆盖;源 `.same/ignore` 中目标尚无的规则会逐条追加:子工作区规则被限定到对应子路径;外部工作区规则没有可推断的子路径,会按目标根语义追加并显示警告。`--move` 仅在全部导入完成后,确认源 `.same` 自导入以来未改变,才删除源状态;源目录中的普通文件不删除。成功默认安静,`--summary` 在 stderr 显示来源、模型、缓存、设置与遥测计数。跨数据库不构成单个原子事务,但每个导入步骤可重试且按来源版本/运行 ID 去重;失败时不会执行 `--move`。绝对路径缓存保留外部目录知识,普通 `scan` 的重复结果仍只包含本轮扫描范围,缓存复用前必须匹配完整文件戳。模型来源再次训练后重合并会替换该来源旧贡献,不会重复累加;模型身份仍须匹配设备与配置才会用于自动路由。遥测仍受目标后续运行的保留策略影响。 + +`merge` imports provenance-keyed model contributions, absolute-path digest cache entries, and telemetry deduplicated by run ID. The destination configuration is preserved; missing ignore rules are appended, with descendant rules scoped to their relative subtree; external rules apply at the target root with a warning. Recursive mode discovers descendant workspaces, and guarded `--move` deletes source state only after successful import and an unchanged-state check. Ordinary source files remain untouched. Cross-database import is retryable rather than one atomic transaction. External cache records remain outside the current scan result set and require an exact current file stamp before reuse. Later source updates replace prior contributions by revision; telemetry retention still follows the destination policy on future runs. + +## 初始化与清理 / Initialize and clean `same new` 创建 `.same/config.toml`(写全本机实际默认值)与 `.same/ignore`(推荐规则),**保留已有文件,不覆盖用户设置**。推荐规则排除版本控制元数据与操作系统目录元数据;`build/`、`node_modules/`、`.venv/` 仅以注释提供,不默认隐藏可能需要去重的内容。无需先运行 `new` 才能扫描:缺少配置时扫描使用内建默认值。 @@ -128,6 +157,8 @@ same --summary > matches.tsv 2> profile.log # 结果与统计分离 / separate r Summary 分区报告及下文所有性能字段均须 `--summary` 才输出;该开关不隐藏警告或错误。 / The sectioned Summary report and all statistics below require `--summary`; warnings and errors remain independent. +`same --help` 也使用相同的颜色策略:交互终端用颜色区分标题、命令与选项;重定向、`NO_COLOR` 或 `--color=never` 保持纯文本。可用 `same --color=always --help` 显式输出带 ANSI 颜色的帮助。 / Help uses the same color policy: colored hierarchy on terminals, plain text when redirected or disabled, with `--color=always` as an explicit override. + `--format=auto|pretty|tsv` 与 `--color=auto|always|never` 相互独立,均为命令行选项,不写入 TOML。自动颜色遵循非空 `NO_COLOR`、`TERM=dumb` 和各标准流是否连接终端;重定向默认无色并保留原 TSV。Windows 自动尝试启用虚拟终端(Virtual Terminal, VT)处理,不支持时降级无色,退出时恢复控制台模式。显式 `always` 会覆盖环境提示,即使重定向也输出 ANSI 控制码;`never` 始终无色。stdout 与 stderr 独立检测:自动模式下,重定向结果不影响终端中的 Profile,重定向统计也不会带入颜色。`--format=tsv` 始终保留原始统计字段;`--format=pretty` 显式选择可读报告与智能单位,重定向时默认仍无色。 Format and color are independent CLI-only options. Auto color honors nonempty `NO_COLOR`, `TERM=dumb`, and independent stdout/stderr terminal detection. Redirected output stays plain legacy TSV. Windows enables VT processing when supported and restores the console mode on exit. Explicit `always` overrides environment hints, including redirection; `never` suppresses all color. Auto mode formats/colors each stream independently: redirecting results does not disable a terminal Profile, and redirecting diagnostics keeps raw uncolored metrics. `--format=tsv` preserves raw profile fields; `--format=pretty` explicitly selects human-readable units, still uncolored by default under redirection. diff --git a/docs/contextual-learning.md b/docs/contextual-learning.md index 431c9b4..e308284 100644 --- a/docs/contextual-learning.md +++ b/docs/contextual-learning.md @@ -48,8 +48,10 @@ The iGPU is admitted without waiting; forced accelerator modes remain subject to `.same/model.db` 独立于摘要缓存 `state.db` 和遥测日志 `telemetry.db`。设备按需初始化后,才按真实设备特征加载匹配的不可变启动快照;身份键还包含特征版本和工作线程数。驱动、实现或相关参数变化导致独立状态,而不是静默混用。 The model database is independent of digest and telemetry databases. Lazy device initialization resolves the exact identity before loading priors. -启动先验在运行边界乘以 0.9;每个工作线程单独累加本轮增量。成功扫描收尾按身份键**只合并一次先验**,再加各线程增量,避免工作线程数倍增历史证据。衰减针对统计权重,不意味着原始样本计数按比例减少,也不是按墙钟时间遗忘。存储故障形成诊断,不改变哈希与分组正确性。 -At run boundaries, prior statistical weight decays by 0.9. Finalization merges one prior per key plus worker deltas, rather than multiplying history by worker count. Raw counts are not effective decayed weight. +启动先验在运行边界乘以 0.9;每个工作线程单独累加本轮增量。成功扫描收尾只汇总各线程的本轮增量;模型库存储各来源的贡献,将衰减应用到被本轮触及的每份贡献,再把本地增量写入当前工作区贡献,重新物化总量。这既避免把先验按线程数重复计入,也允许再次合并已继续训练的来源时按版本替换,而非重复累加。衰减针对统计权重,不意味着原始样本计数按比例减少,也不是按墙钟时间遗忘。存储故障形成诊断,不改变哈希与分组正确性。 +At run boundaries, touched provenance contributions decay by 0.9. Finalization combines only current worker deltas, updates the local contribution, and rematerializes the aggregate. This avoids multiplying a prior by worker count and permits revision-based replacement on remerge. Raw counts are not effective decayed weight. + +当前实现只保存本轮有新观测的设备键。因此某设备本轮没有新样本时,数据库中的旧统计量不会被改写;下轮加载时仍只对同一份旧值乘一次 0.9。这里的“每轮衰减”不应理解为无观测运行也会逐轮累计老化,这可能让长期未使用设备的先验保持陈旧。 / The current implementation saves only keys with new observations in this run. A backend with no new sample keeps its stored statistics unchanged; the next load applies one 0.9 factor to the same stored value, not cumulative aging across unobserved runs. This is a stale-prior limitation, not elapsed-time forgetting. | 开关 / Switch | 学习与 model.db / Learning | telemetry.db | |---|---|---| @@ -80,8 +82,8 @@ Prioritize controlled interleaved comparisons and ablations over utilization alo ## 冷启动准入 / Cold-start admission -设备探测(probe)与激活(activation)分离:探测建立设备身份与能力,激活才创建执行资源、编译内核并检查正确性。探测本身仍可能调用驱动、产生耗时,不能称为免费。初始化历史独立于稳态服务模型,保存在模型库 schema 2;不能把初始化时间作为每个 payload 的哈希服务标签。 -Probing identifies capabilities; activation creates executable resources and checks correctness. Probe costs are not necessarily zero. Schema 2 stores setup history separately from steady-state service statistics. +设备探测(probe)与激活(activation)分离:探测建立设备身份与能力,激活才创建执行资源、编译内核并检查正确性。探测本身仍可能调用驱动、产生耗时,不能称为免费。初始化历史独立于稳态服务模型,保存在模型库当前 schema 4;不能把初始化时间作为每个 payload 的哈希服务标签。 +Probing identifies capabilities; activation creates executable resources and checks correctness. Probe costs are not necessarily zero. Current schema 4 stores setup history separately from steady-state service statistics and includes a provenance ledger. 自动且启用 PGO 时,只有临时决策实际请求某设备,才执行其冷启动预检(preflight)。共享非阻塞门槛要求:当前 CPU 任务的已知保守成本至少覆盖 bootstrap 估计,或本轮探索信用覆盖该估计,才允许 OpenCL ICD 探测或 CUDA 创建。竞争失败直接继续 CPU,不等待;静态 no-PGO 决策为 CPU 时不探测设备。首次未知且很短的扫描因此有意保持 CPU。`cuda_bootstrap_ms=100.0` 是 CUDA 的独立默认估计。 Automatic PGO preflights only a provisionally requested device, before ICD discovery or CUDA creation. A shared nonblocking gate requires known CPU-task potential or funded credit; unknown short scans intentionally remain CPU. This is not guaranteed optimal under unknown cold costs. diff --git a/docs/merge-design.md b/docs/merge-design.md new file mode 100644 index 0000000..b450c1c --- /dev/null +++ b/docs/merge-design.md @@ -0,0 +1,94 @@ +# Workspace merge design (2026-09-27) + +## Decision and invariants + +The user requested `same merge [dir] [-r] [--move] [--summary]` to combine telemetry, +online learning, content-cache knowledge, and missing ignore rules into the current +workspace while preserving its configuration. Recursive mode discovers descendant +`.same` workspaces without following links. The default source directory is `.`; +the current workspace itself is never imported into itself. `--move` removes source +workspace state only after every requested import succeeds and is verified. + +Clarifications: cache keys should be absolute paths, so unrelated directories can +contribute reusable entries; ordinary scan results must still contain **only files in +the current scan scope**. A source that is trained again and merged again must +**replace its previous model contribution**, not add the entire cumulative state a +second time. These choices supersede an earlier proposed relative-prefix mapping. + +### State cache + +`state.db` schema v1 stored root-relative paths in `files` and deleted unvisited +records at `end_scan`. Changing this table in place to an all-workspace archive would +make duplicate output include external paths and would break callers. Schema v2 +adds an `absolute_cache` table keyed by canonical absolute file path; it keeps +`files` as the current-scan projection. Merge reads source relative records and +absolute archive rows, converts relative records using the source root, and imports +them even if files are temporarily absent. Scan checks the complete current +`FileStamp` before reusing an absolute entry on an active-cache miss, then materializes a relative +record in the current scan; comparison candidates remain active-scope only. +The file stamp is a conservative cache key, **not** proof against an adversary who +changes content while preserving all metadata. No imported digest may bypass the +existing bytewise duplicate verification. + +### Online model + +The original `models` table stored already-aggregated, run-decayed sufficient +statistics. Blindly adding two such rows is not idempotent, and choosing the row +with more samples discards independent evidence. Schema v4 adds a provenance ledger +whose rows are keyed by stable workspace identity and model identity key. Keep the +existing `models` table as a materialized aggregate for current readers. Merge +replaces a source's ledger row and recomputes the aggregate. Local scan/train +updates must also update the local contribution, rather than writing an opaque +aggregate that would be impossible to decompose on later source replacement. +Run-boundary decay must be applied consistently to every contributing component +for any key touched by a local run. Setup histories need corresponding provenance +and recomputation. Migrating a v2 database treats its existing aggregate as one +local contribution. Each workspace instance has a random identity in addition to +its canonical path. A copied database retains imported contributions, but when +opened at a new root it forks its local identity. Recreating `.same` at the same +path after `--move` also gets a new identity, preventing stale revision collisions. +The ledger is bounded by the existing 32 model identities, which can evict old +device/configuration keys; this is not an unlimited cross-machine repository. + +### Telemetry, ignore, and deletion + +Telemetry import is by immutable run ID, with related events, metrics and +parameters copied together; a repeated merge must not duplicate runs. Existing +writer retention may later remove old imported runs according to the destination +configuration, so the merge report must not promise indefinite archival. +Missing destination ignore rules are appended one by one. Descendant-source rules +are rewritten as target-root-anchored patterns under the source relative path; +external-source rules cannot be scoped and therefore apply at the destination root +with an explicit warning. Existing target rules keep their order, new rules are +appended in source order, and a repeated +merge does not append exact duplicate translated lines. Configuration remains untouched. +Destination `.same/config.toml` is never copied or overwritten. Import operations +must be idempotent across partial failures. Source deletion is deferred until all +database connections are closed and the complete import succeeds; any inability +to prove safe deletion leaves sources intact and reports failure. The source-state +snapshot checks direct child names, identity, timestamps, size, and BLAKE3 contents +while under a workspace lock. A non-cooperating external process can still race a +filesystem operation; `--move` is not an adversarial-filesystem security guarantee. +With multiple sources, deletion is not globally atomic: a later failure can leave +earlier sources removed, but the imported data remains in the destination. + +SQLite transactions are per database here. WAL-mode transactions across attached +databases do not provide all-database crash atomicity, so imports are independently +transactional and designed to be retryable. The telemetry first-copy path uses the +[SQLite backup API](https://www.sqlite.org/backup.html); the cross-database limit is +documented in [SQLite ATTACH](https://www.sqlite.org/lang_attach.html). The +[SQLite WAL documentation](https://sqlite.org/wal.html) explains the associated +journal-mode behavior. This is an engineering constraint, not a claim that a full +merge is one transaction. + +## Validation evidence and remaining limits + +Debug tests cover multi-root selection and overlap rejection, model remerge and +continued training, copied and recreated workspace identities, telemetry run-ID +deduplication, scoped ignore rules, destination config preservation, active-scan +result isolation, actual absolute-cache reuse without rehash in a parent scan, +recursive move, active source locks, and same-size modification of source state +before cleanup. A physical CUDA/iGPU multi-root training run was also performed +on three selected files; see `docs/train-mode-validation.md` for +hardware-specific details. These tests do not establish general scan speedup. +Experiments and merge fixtures are kept under repository `.temp`/`.cache`. diff --git a/docs/research/online-model-parameter-audit-2026-09-27.md b/docs/research/online-model-parameter-audit-2026-09-27.md new file mode 100644 index 0000000..ff602b8 --- /dev/null +++ b/docs/research/online-model-parameter-audit-2026-09-27.md @@ -0,0 +1,58 @@ +# Online model and initialization-parameter audit (2026-09-27) + +## Scope and evidence + +This audit describes the current repository source, not a new performance experiment. It separates the contextual service model, legacy band API, routing heuristics, and cold-start admission. Primary local sources: `include/same/detail/online_model.hpp`, `src/online_model.cpp`, `include/same/resources.hpp`, `src/resources.cpp`, `include/same/config.hpp`, `src/application.cpp`, `docs/contextual-learning.md`, `docs/contextual-model-performance.md`, and `docs/contextual-scan-performance.md`. No user filesystem or production model database was inspected. Existing validation results are cited rather than rerun. + +## Mathematical contract + +Each content worker owns a separate four-feature model for each of CPU, CUDA, and iGPU. For bytes B, effective batch L, and observed peer count C, x = [1, B/2^20, ceil(B/L)/1024, (B/2^20)C/1024]. The response is successful full-file hash service time in milliseconds, including file reads and device/resource contention, not kernel-only duration or scan makespan. CPU/iGPU C counts host-side CPU+iGPU work; CUDA C counts active CUDA work. Only the selected, successful, unretried backend provides a label. Eligible tasks are profiled; below the 16 MiB default accelerator floor, one in 64 tasks per worker is profiled. + +Per backend, S = sum(xx^T), t = sum(xy), q = sum(y^2), W = sum(weights). D_ii = sqrt(max(S_ii, 1e-24)); a Cholesky solve computes (D^-1 S D^-1 + 1e-8 I) theta = D^-1 t and beta = D^-1 theta. First four run observations trigger each-fit updates, then every 16 observations. Prediction is x^T beta if finite and positive, with an in-sample RMS residual and a coordinatewise training-range flag. Invalid numerical state or a failed solve does not publish a new fit. The older 32 factor-four size-band, alpha=1/8 per-byte EWMA API still exists but the current automatic choice calls the contextual overload. + +At run startup, matching device/CPU identity plus worker-count history is loaded from `.same/model.db`, multiplied by 0.9 in sufficient-statistic weight, and copied into worker-local models. The raw sample count is not decayed. At successful shutdown the prior is merged only once per identity key, plus each worker's run delta. A new identity starts without synthetic performance priors. No within-run age weighting is present: all accepted observations during one run enter S,t,q with weight one. Thus 0.9 is a per-run forgetting rule, not a five-sample or wall-time half-life. + +Important implementation qualification: `save_learning` skips any backend identity with zero new observations in the run. That identity's stored state is not written back, so successive CPU-only runs do **not** compound 0.9 decay on an unused GPU prior. Each later load applies 0.9 afresh to the unchanged stored statistics. The ~6.58-run half-life below applies only to a history key repeatedly saved after new observations; it is not a universal age guarantee for inactive backends. + +Routing first applies accelerator size/budget/availability checks. Default `gpu_min_bytes` is 16 MiB; below it the CPU is mandatory. With unknown service models the static preference is CUDA (then iGPU) only from 64 MiB, otherwise CPU. Per-worker size-band first exploration attempts are capped at two per candidate, with periodic deterministic attempts every 64 eligible arrivals. For known in-domain predictions, an accelerator is chosen only if its predicted service plus in-sample RMS is below 0.95 times CPU predicted service minus CPU RMS. This is not a calibrated confidence comparison. Device startup is separately gated by a 100 ms CUDA/iGPU default bootstrap estimate, historical iGPU maximum setup cost when available, or run-local credit equal to 0.05 times completed eligible CPU service divided by worker count minus cold expenses. The gate is best effort, not a hard startup cost bound. With four workers and no other cost or known saving, funding a 100 ms estimate requires about 8,000 ms of cumulative eligible CPU service (100*4/0.05); with eight workers, about 16,000 ms. + +## Parameter judgment and limitations + +| Parameter | Judgment | Evidence / caveat | +|---|---|---| +| Four-feature linear form | Reasonable low-cost local baseline, not validated as a causal or generally calibrated cost law | Batch count and bytes can be nearly collinear with fixed L; I/O cache state, startup, remaining workload, and interaction effects are unmodeled. Device properties partition state rather than supplying transferable regression features. | +| Normalized ridge 1e-8 | Numerical safeguard, not empirically tuned regularization | Diagonal scaling changes interpretation versus a fixed physical-unit penalty. Existing collinearity test establishes a solvable case, not prediction calibration under sparse/shifted data. | +| First 4 / then every 16 fitting | Sensible bounded CPU overhead, not proven optimal adaptation cadence | Fit can be based on one observation, whose in-sample residual may be near zero. A full run gives equal weight to old and new samples, so a within-run regime shift may adapt slowly regardless of refit cadence. | +| Per-run decay 0.9 | Plausible conservative persistence policy, not evidence-backed optimum | At equal observations per run, old-run weight half-life is ln(0.5)/ln(0.9) ~6.58 runs; many new observations in a single run still receive no within-run forgetting. Scan frequency is not elapsed time. | +| RMS and 5% CPU margin | Useful conservative heuristic but no uncertainty guarantee | RMS is training error; selection-dependent labels and feature-domain checks do not establish out-of-sample error coverage. The margin should be evaluated using held-out/pre-decision errors and end-to-end regret. | +| 16 MiB eligibility / 64 MiB cold preference | Defensible protective starting defaults, not universal crossover sizes | The final local short-workload study showed that default small files avoid discovery; it did not demonstrate acceleration on sustained large jobs. | +| 100 ms setup fallback / 5% credit | Conservative initial admission heuristic, hardware-dependent | A deliberately optimistic 1 ms physical iGPU test measured ~354.620 ms setup; its next process reused the measured setup cost and deferred activation. The default 100 ms is not a safe upper bound or measured device-specific prior. | +| 2 initial attempts / 64 periodic arrivals | Bounds exploration pressure, but proof of adequate coverage is absent | Exploration is worker-local and deterministic, may fragment evidence, and only selected-device outcomes are observed. Cold-start gates can suppress attempted exploration. | + +The final local preflight-gated study (Windows/i9-12900H, two warmups and seven measured rounds per condition, 74 correct scans) reported default-floor small median wall times of 104.147 ms for auto+PGO versus 96.225 ms for auto/no-PGO, and mixed 101.664 versus 105.399 ms. Neither automatic condition discovered or initialized iGPU on these short workloads; therefore these measurements support avoiding wasteful startup, not a GPU speedup or parameter optimality. CPU+PGO was 106.571 versus CPU/no-PGO 94.723 ms on small tasks; persistence accounted for several milliseconds. Earlier negative results are preserved in the same report. The hot single-thread model microbenchmark reported ~20.923 ns per contextual prediction and ~116.541 ns per amortized observation, but does not measure full scheduling, contention, or scan speed. + +## Selected-feedback starvation (follow-up) + +CPU observations never directly update CUDA/iGPU coefficients. The risk is opportunity starvation, not numerical contamination: when a provisional GPU exploration is rejected by cold admission, execution falls back to CPU and no GPU service label is generated. The initial per-band exploration counter increments only after successful backend activation/selection (`src/resources.cpp`, `select_backend`), so a denied GPU attempt remains eligible for a later retry. Periodic exploration every 64 *eligible arrivals per worker and size band* also remains. Neither mechanism guarantees a completed GPU observation: both pass through the same 100 ms or history-based cold gate, and run-local credit resets next process. Below the 16 MiB eligibility floor, GPU observations are intentionally impossible. Thus a series of short scans can retain only CPU labels indefinitely even when a GPU might have been beneficial on some eligible shape; conversely this conservatism can correctly avoid setup costs that exceed remaining work. Which interpretation holds requires a counterfactual experiment, not CPU-only logs. + +Reproducible diagnostic: use one controlled eligible size band and `--rehash`, record per-run CPU/CUDA/iGPU accepted samples, exploration selections, deferred CUDA/iGPU counts, cold credit/spend and device setup, and compare default auto with a separate forced-device control on the same bytes and randomized execution order. Verify complete outputs and include startup in wall time. A nonzero number of exploration *decisions* without completed device samples is not evidence that the model has learned the device. Do not add unconditional GPU probing solely to make data symmetric: it may regress short scans. If the controlled long-lived workload shows missed benefit, consider a budgeted, shared, size-stratified completed-probe guarantee rather than merely increasing provisional exploration frequency. + +An explicit `same train` command now offers an opt-in way to pay for all three backend labels outside ordinary scans; implementation and bounded physical-device smoke are recorded in [train-mode-validation.md](../train-mode-validation.md). Its existence does not imply that an auto scan should proactively start accelerators, nor that its selected corpus represents a production workload. + +## Lazy versus asynchronous activation (follow-up) + +These are independent choices. Lazy activation controls **whether/when** the process pays GPU setup; asynchronous activation controls **whether that paid setup overlaps useful CPU/metadata work**. `Resources` is constructed before `scan()`, but `scan()` learns which files actually miss the hash cache at `src/application.cpp` lines 601-623. Therefore unconditional asynchronous setup at resource construction would still charge cache-hit and all-small scans. The original lazy design deliberately avoided that cost; earlier short-scan experiments observed hundreds of milliseconds of wasted iGPU discovery/setup, later preflight gating eliminated it for the final short fixture. + +The current first-demand activation is synchronous on the selecting worker, while other workers continue CPU work. CUDA's primary context is process-shared, but worker streams, buffers and backend objects are created and destroyed on their owner threads (`src/resources.cpp`, `prepare_gpu`, `run`; `src/compute_cuda.cu`). OpenCL program/context creation and source compilation are shared and currently synchronous; the single pool iGPU backend has its own admission and handoff invariants (`src/compute_opencl.cpp`, `program_for`; `src/resources.cpp`, `select_igpu`). A background thread may safely prewarm a shared context/program only with explicit publication, error and shutdown contracts; it must not silently move worker-owned CUDA objects or omit correctness validation. CUDA documents primary-context sharing across host threads and runtime initialization on `cudaSetDevice` in CUDA 12+, but not zero setup cost: https://docs.nvidia.com/cuda/cuda-programming-guide/02-basics/intro-to-cuda-cpp.html . Khronos documents that `clBuildProgram` with a null callback waits for compilation, as used here: https://registry.khronos.org/OpenCL/specs/unified/refpages/man/html/clBuildProgram.html . + +Recommended experiment before implementation: compare (A) current first-demand lazy setup, (B) unconditional startup background setup, and (C) budgeted async prewarm triggered by the first eligible cache miss or a bounded backlog. Preserve forced CPU/no-PGO semantics and default output correctness. Measure process wall time, scan/metadata/hash critical path, discovery/setup duration and overlap, driver-induced CPU contention, initialized devices with zero useful hashes, memory, backend work, and failures. Include all-cache-hit, all-small, one-large-file, and sustained-large-file cases. Background launch is beneficial only when saved critical-path latency plus subsequent GPU service savings exceed resource contention and unnecessary setup. Do not hide the setup time by moving it outside the reported scan/process interval. + +## Highest-value next experiment + +Do not change defaults based on this audit alone. Use a fixed binary and ordinary generated files under repository `.cache` or `.temp`; force rehash, isolate `.same` state per arm, validate complete digests/groups, and interleave randomized order. Stratify payloads (4 KiB, 1 MiB, 16 MiB, 64 MiB, 512 MiB), cache state, worker count, device startup state, and within-run speed drift. Compare current defaults against a small preregistered parameter grid: eligibility 16/64 MiB, startup fallback 100 ms versus measured setup history, decay 0.9 versus a bounded within-run forgetting variant, and RMS margin versus held-out-error-calibrated margin. Record backend choice, setup/discovery costs, pre-update residuals by backend and size, known/out-of-domain rate, PGO lifecycle time, end-to-end wall time, and peak memory. Include forced CPU, static/no-PGO, and forced available devices as controls. A single-machine optimum must not be exported as a universal default. Parameter changes must preserve external CLI/config compatibility. + +## External anchors + +- StarPU's official performance-model documentation and tutorial describe continuously calibrated, per-architecture history models and stale-calibration reset; this supports runtime observation as an engineering pattern, not direct transfer of StarPU's task-graph performance to this scanner: https://starpu.gitlabpages.inria.fr/tutorials/2021-02-EoCoE/ . +- Li et al., *A Contextual-Bandit Approach to Personalized News Article Recommendation*, WWW 2010, motivates the selected-action feedback framing, not a LinUCB guarantee for this deterministic policy: https://www.schapire.net/papers/www10.pdf . +- Russac et al., *Weighted Linear Bandits for Non-Stationary Environments*, NeurIPS 2019, examines discounting under drifting reward models; its assumptions and regret bounds are not established here: https://arxiv.org/abs/1909.09146 . diff --git a/docs/summary-output-design.md b/docs/summary-output-design.md index 5166902..c8697c5 100644 --- a/docs/summary-output-design.md +++ b/docs/summary-output-design.md @@ -83,3 +83,14 @@ Tests should establish all three layers independently: Visual checks should cover a normal run, hidden unique files, disabled PGO/telemetry, a warm cache, and a writer error. Timing values are nondeterministic and must be validated by names, units, and accounting invariants rather than golden snapshots. + +## CLI help hierarchy + +The help page uses the existing stdout `TerminalColor` policy rather than a separate color +implementation. Magenta marks the product identity, cyan marks section headings, and green marks +command/option labels; indentation and literal headings preserve the full hierarchy when ANSI is +disabled. Usage, commands, scan-only options, train-only options, shared options, and notes are +separate groups. Auto color remains TTY/`NO_COLOR` aware, while an explicit `--color=always` may +color redirected help for inspection. The CLI accepts this override before or after `--help`. +Help does not load configuration or create workspace state. A redirected integration test checks +both forced-color orderings and the `--color=never` plain-text path. diff --git a/docs/telemetry.md b/docs/telemetry.md index 92c4e8a..1cf8f1e 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -69,6 +69,14 @@ Schema version is 1 with application ID 1396788564 (`0x53414D54`). Ownership/ver Run accepted/persisted/dropped counters cover queued events only, not reserved final stages/metrics/parameters. Exact final drain includes DB close/thread exit and is available in current-run Summary after joining, not falsely persisted before it is known. Inspect both truncation totals and row flags. +### 显式训练的端到端记录 / End-to-end train records + +`same train` 默认复用同一遥测库,`runs.command='train'`;`root` 是模型所属工作区,`config_json` 内的 `corpus` 是只读语料绝对路径,另含有效配置与选样上限。`runs.status` 对完整成功记 `completed`,设备不可用、文件变化、摘要不一致或保存失败记 `failed`,具体异常保留在 `runs.error` 和 `train.failed` 日志。`--no-telemetry` 不创建本轮遥测记录,但模型仍训练并落盘。 / Train uses the existing database with `command='train'`, a workspace root and an explicit corpus path in `config_json`. Failed training is finalized as failed with an error and terminal log; no-telemetry leaves training/model persistence active. + +固定终态跨度为 `train.select`、`train.initialize`、`train.measure`、`train.persist`、`train.output` 及父跨度 `train.run`。终态指标包括发现/选中/达到 GPU 资格/完成三方校验的文件数、逻辑输入及实际应用层读字节、各后端提交与接受样本数、预测前误差、先验命中与保存键数、CUDA 各线程启动时间之和、iGPU 发现/初始化时间、阶段与运行时间。`train.backend_attempt` 是每后端每文件的普通有界跨度,包含请求后端、工作线程、大小、起点、耗时及成功/失败;不包含文件路径或内容。其 `span_id=1000000+file_index*3+backend_index`,后端顺序为 CPU=0、CUDA=1、iGPU=2。普通事件可能被丢弃,不能只数跨度断言完成量;用终态 `train.*_samples`、`train.verified_files` 与 `runs.dropped/errors` 联合核对。 / Five fixed stages and the run span use the terminal slot; bounded per-backend attempts may be dropped. Final `train.*` metrics are the complete count source. A failed run can have partial accepted in-memory samples without publishing training deltas to the model store. + +库内 `train.elapsed_ms` 截止输出完成,不含遥测写入器的最终排空;stderr 单列 `telemetry_drain_ms` 与 `total_including_telemetry_ms`。CUDA 启动值是线程时间之和,可能与测量阶段重叠,不能与阶段或进程墙钟直接相加。语料路径及失败错误仍可能敏感,即使逐任务跨度省略路径。 / The stored elapsed value ends before telemetry drain; stderr reports drain and total including drain. Summed CUDA setup is not additive wall time, and corpus identity/error text can be sensitive. + 索引覆盖 `(run_id,time_ns)`、`(name,run_id)`、`(run_id,span_id)` 和运行开始时间;指标、参数按运行与名字建立主键。外部客户端建议以只读模式打开,并保持短读事务。仅复制主 `.db` 可能遗漏 WAL 中尚未合并的数据;运行中备份应使用 SQLite 备份 API 或客户端备份功能。 Indexes cover run/time, event name/run, run/span and run start. Metrics and parameters have run/name keys. Prefer read-only clients and short transactions. Copying the main DB alone can omit live WAL data; use SQLite backup facilities for live backups. @@ -142,6 +150,8 @@ Hash spans reuse analyzer sampling: eligible work is sampled, small work once ev ## 生命周期、保留与隐私 / Lifecycle, retention and privacy +`same merge` 按运行 ID 导入来源工作区的遥测运行及其事件、指标和参数;重复合并跳过已导入运行。导入不覆盖目标配置,也不保证永久保留:目标下一次运行仍会按自己的保留配置淘汰旧运行。`merge --summary` 报告本次实际新增的运行与事件,不等于数据库当前总数。/ `same merge` imports telemetry by run ID with dependent rows, skips already imported runs, and preserves destination configuration. Later destination retention can evict imported runs. The merge summary counts newly imported rows, not the database total. + 后台打开前检查主库与 `-wal`、`-shm`、`-journal` 侧文件:拒绝符号链接、Windows 重解析点(reparse point)、多硬链接与非普通文件。该保护用于拒绝已存在的不安全路径,不声称抵御拥有同目录写权限的恶意进程并发替换路径。检查失败仅禁用遥测并报告,不阻塞文件比较业务。 Before opening, the writer rejects existing symlinks, Windows reparse points, multiply hard-linked files and special files at the main DB and its -wal/-shm/-journal paths. This rejects preexisting unsafe paths, not malicious concurrent replacement by a process able to write the directory. Failure disables telemetry, not file comparison. diff --git a/docs/train-mode-validation.md b/docs/train-mode-validation.md new file mode 100644 index 0000000..c41df25 --- /dev/null +++ b/docs/train-mode-validation.md @@ -0,0 +1,77 @@ +# Explicit three-backend training: design and validation (2026-09-27) + +## Decision and contract + +`same train [options]` separates the cost of obtaining CPU/CUDA/iGPU labels from ordinary automatic scans, **not** the training model from the serving model. Zero positional directories default to the current working directory; explicit directories are read-only, while the current working directory owns `.same/model.db` and the run lock. Subsequent automatic scans launched in that same working directory load and continue updating the same model. The legacy `--corpus=DIR` syntax remains an alias for adding a corpus directory. All directories share one selected-file and logical-byte budget; overlapping roots are rejected. The existing scan cache `state.db`, duplicate groups, and input contents are not written by training. Training is opt-in and forces all three backends for every selected nonempty file, even below the normal GPU eligibility floor, but requires at least one selected file at or above that floor. A missing/failing backend or digest mismatch fails the run without publishing its training deltas. The model ledger still uses per-key atomic replacement rather than one transaction across all three keys; a storage failure during final persistence can leave a subset of keys updated, and is reported as failure. + +Metadata selection walks the entire corpus using the existing link-rejecting, ignore-aware `ParallelWalk`. It retains up to 64 deterministic path-ranked candidates per factor-four size band, then selects round-robin across bands under the configured global file/byte ceilings (defaults 128 files and 2 GiB logical corpus bytes; maximum 4096 files). The metadata walk is not bounded by the selected-file count. Each selected file runs through the automatic mode's actual pool budgets and device profiles rather than the forced-scan mode's potentially different batch capacity. Three backend requests are submitted together, rotating order between files; the main thread waits for all complete digests, the resource pool's full idle state, and their equality before the next file. The training reader is reopened before timing so the service label covers the same version checks, full-file read, hash and finish as the scan's metadata-handle reuse path. Startup/discovery costs remain separate from service labels. The training model is then merged by the existing device/CPU identity key and worker count. Training intent is passed through a new call overload, preserving the old `Worker` layout and existing three-argument selection symbol. + +This is a supervised calibration experiment on selected actions, not evidence of an end-to-end speedup. Three reads of the same file alter OS cache state; rotating order reduces systematic first-position bias but does not remove storage/cache or thermal confounding. A backend may implement some tiny payload work on the host; a valid backend service sample is not proof that a GPU kernel executed for that payload. Few eligible files, narrow sizes, and low contention can leave a rank-deficient or out-of-domain model. The command warns when fewer than four selected files satisfy GPU eligibility. Real scans continue to supply online updates, but a large training corpus can initially outweigh a few real observations because its sufficient statistics are persisted at full sample weight. This requires held-out scan validation; the train command does not certify model quality. The original file hash oracle is the CPU backend plus full cross-backend digest equality, not an independent cryptographic BLAKE3 proof. + +## Reproduction and observations + +Source base was `331164a` plus the uncommitted train, telemetry, and report implementation. Windows x64, Visual Studio 2022 MSVC 19.44, CUDA 12.8 and dynamically loaded OpenCL were used for the physical smoke. Training-validation Release executable SHA-256: `C4722A2EDD3A8F93532D79B4B560F3AC0E432D645B1C71FF9EA9F3B42AAA03AD`; a later help-presentation-only rebuild has SHA-256 `6C662AC40FCC56486969E3BE3035644C5C059B2D38A41FEE163A2C59ADDF96D6`. CMake dependencies came from repository `.cache` source checkouts; no external user directory was read. This is a correctness/lifecycle smoke, **not** a benchmark: no controlled repetitions, ordering randomization, power/frequency controls or cold-cache reset were performed. + +Isolated fixture beneath repository `.temp`: workspace and separate corpus; four ordinary files of 4 KiB, 64 KiB, 1 MiB and 16 MiB, generated with a fixed PRNG seed 42. Workspace config: workers 2, metadata workers 2, CPU block 1 MiB, host budget 128 MiB, device budget 64 MiB, queue capacity 4, `gpu_min_bytes` 1 MiB, backend auto. Invocation from the workspace: + +```powershell +same train --corpus=../corpus --max-files=4 --max-bytes=33554432 +``` + +Observed: exit 0; 4 accepted samples each for CPU, CUDA and iGPU; 17,895,424 logical input bytes; three model keys saved. Repeated training and an intervening ordinary auto scan preserved the three-key success. The final-source physical smoke reported CUDA setup sum 108.133 ms, iGPU discovery 121.603 ms and iGPU setup 325.956 ms. These are separate diagnostics, not additive wall times or stable performance estimates. The input corpus SHA-256 checked before/after remained unchanged; a fresh training workspace had no `state.db`. After a normal `same scan --rehash --summary --no-telemetry` on a copied 16 MiB input, the scan reported two CPU prior hits for its two workers and no invalid model state. Later training left the scan's `state.db` SHA-256 unchanged. This proves matching CPU-state reuse, not that the GPU prior was selected by that scan. The fixture is intentionally too small for a model-quality or throughput conclusion and emitted the sparse-eligible warning. + +The final successful physical run also verified the telemetry SQLite database directly: `latest_run` had `(command=train,status=completed,accepted=12,persisted=12,dropped=0,errors=0)`. The spans contained one successful `train.select`, `train.initialize`, `train.measure`, `train.persist`, `train.output` and `train.run`, plus four `train.backend_attempt` events per backend. The terminal log was `train.completed`. Final metrics included `train.selected_files=4`, `train.verified_files=4`, `train.model_saved_keys=3` and `train.read_bytes=53,686,272`, exactly three times the selected logical bytes. Stored pre-drain `train.elapsed_ms=495.7609`; stderr separately reported `telemetry_drain_ms=4.8364` and `total_including_telemetry_ms=500.697`. These are one smoke run, not performance evidence. + +The instrumentation uses the existing local SQLite writer, not an OpenTelemetry exporter and not a claim of OpenTelemetry schema compliance. Its run/phase/attempt hierarchy follows the useful distinction between root spans, child operation spans, terminal metrics and logs in the [OpenTelemetry tracing API](https://opentelemetry.io/docs/specs/otel/trace/api/) and [semantic-convention overview](https://opentelemetry.io/docs/concepts/semantic-conventions/). Per-attempt spans are bounded by the configured queue/event cap; terminal aggregate metrics and explicit dropped/error counters remain the reliable source for completeness. This trade-off is also motivated by research on information-preserving trace sampling, such as [STEAM (ESEC/FSE 2023)](https://www.microsoft.com/en-us/research/publication/steam-observability-preserving-trace-sampling-2/), but no adaptive sampling algorithm was adopted here: this is a local, small-batch command, and the immediate requirement is to make loss observable rather than add sampling complexity. + +A CPU-only MSVC Debug build produced `training backend unavailable: cuda`, exit 2, no `state.db`, unchanged corpus length and an empty newly created model database. Hardware-free `training_tests` asserts zero model rows after failure, a finalized `failed` telemetry run, `train.measure` error span, `train.failed` log and complete `train.model_saved_keys=0` metric. Its disabled-telemetry variant creates no telemetry database. `igpu_routing_tests` injects fake CUDA/iGPU backends and verifies forced training ignores the size and cold-credit gates while keeping backend labels distinct. The final focused CTest batch passed 9/9: telemetry, training, iGPU routing, online resources, adaptive routing, hash retry, unified CUDA, scan integration and learning integration. The test temporary directory was redirected beneath repository `.temp`. + +The default-corpus path was exercised separately: a fresh workspace beneath `.temp` held one ordinary 1 MiB input with `gpu_min_bytes=1 MiB`; `same train --max-files=1 --max-bytes=2097152` returned exit 0, one accepted sample for each backend, three saved model keys and no `state.db`. This verifies the requested current-working-directory default, not adequate one-sample model quality; the sparse-eligible warning was emitted. + +On that same isolated workspace, `same train --max-files=1 --max-bytes=2097152 --no-telemetry` also succeeded with one accepted sample per backend and three model keys while `telemetry.db` remained absent. This confirms that the switch disables diagnostics, not model training. + +## Training report validation + +A final physical quiet-output check used a fresh repository-local `.temp/train-silent-report` workspace with four GPU-eligible files. Plain `same train` exited 0 with zero-byte stdout and stderr logs; `same train --summary --format=pretty --color=never` exited 0 with zero-byte stdout and a nonempty sectioned stderr report. Both runs used real CPU/CUDA/iGPU backends and persisted the three model keys. The three directly affected Debug tests (`training`, `integration`, `telemetry`) passed after the quiet-output change. The build kept the pre-existing MSVC LNK4098 default-library warning; no link failure occurred. + +The new `same train --summary --format=pretty --color=never` was run with the real CPU/CUDA/iGPU fixture, exiting 0. After the quiet-output change it emits no stdout result line and a sectioned stderr report using the scan report's existing `PrettyReport`, `human_bytes`, `human_duration`, `render_learning`, and `render_telemetry` helpers. It reported 4 selected/verified files, 4 samples for each backend, 17.07 MiB logical corpus input, 51.20 MiB logical reads across backends, phase times, setup costs, model load/save health, and 12/12 persisted attempt spans. The fixture only had 2 GPU-eligible files, correctly warning that coverage is weak. `--summary --format=tsv --no-telemetry` also exited 0 and emitted raw `train_*` and `model_*` key-value lines followed by `telemetry_enabled=0`; training still saved all three model keys. The report's prediction checks are online residual diagnostics, not held-out policy validation. The old C++ `train` entry point remains available for linkage; successful training is now quiet without presentation options, while warnings and failures remain visible. Following the implementation, the 9-test focused Debug CTest batch passed before a final wording-only report row was added; that row was compiled in the final Release build. + +## Multi-directory and merge smoke (2026-09-27) + +After the provenance-ledger and positional-directory changes, the final Release +Windows CUDA executable had SHA-256 +`91A0502BAB930C28EEB88AEDCCD71E8234896C0FD88346E19A08F96A8D79D34A`. +It was built using Visual Studio 2022 MSVC 19.44 and CUDA 12.8. The linker still +emitted the pre-existing LNK4098 default-library warning; linking succeeded. + +An isolated `.temp/train-multi-final` workspace trained over two repository-local +corpus directories, with three GPU-eligible files totaling 22 MiB logical bytes. +`same train --max-files=3 --max-bytes=33554432 --summary +--format=pretty --color=never` exited 0, wrote no stdout, verified all three +files, recorded three CPU, three CUDA, and three iGPU samples, saved three model +keys, and persisted all 9 accepted trace records with zero drops/errors. The +measured run reported 543.07 ms before telemetry drain and 546.74 ms including +drain, 95.61 ms summed CUDA setup, 123.69 ms iGPU discovery, and 345.58 ms iGPU +activation. Only three eligible files were selected, so it correctly warned +about sparse coverage; these single-run times are not performance estimates. + +A fresh `.temp/merge-final` destination then ran `same merge +../train-multi-final --summary --format=pretty --color=never` using that same +Release binary. It exited 0 with no stdout and reported one workspace imported, +three model contributions and two setup contributions considered, one telemetry +run and 16 event rows newly imported. This source had no scan state cache or +ignore rules, so both relevant import counts were zero. Debug regression tests +separately exercise those paths. The related Debug CTest batch passed 8/8 after +the main merge implementation; the schema-2 migration and first-copy telemetry +recovery changes were each verified by their focused tests afterward. Finally, +the complete CPU-only MSVC Debug CTest suite passed 27/27 in 8.80 seconds, with +`TEMP` and `TMP` redirected beneath repository `.temp`. + +## Follow-up evidence needed + +- More than four eligible files over several size and batch regimes; report pre-update prediction errors and out-of-domain rates separately for all three backends. +- Controlled warm/cold-cache and backend-order crossover. A shared filesystem may be I/O-bound enough that GPU compute advantage is irrelevant. +- Compare CPU/no-PGO, auto before training, auto after training, and forced-device controls with identical input and `--rehash`. Preserve full output correctness and setup-inclusive process times. +- Verify deployment identity changes (driver, binary implementation, worker count or capacity) invalidate the prior, while an identical configuration loads all relevant backends once they are initialized. + +External engineering anchor: StarPU's official performance-model tutorial describes continual calibration and invalidation of stale kernel models, but does not establish this scanner's performance: https://starpu.gitlabpages.inria.fr/tutorials/2021-02-EoCoE/ . Scientific framing: selected-device-only labels are partial-feedback data, so calibration coverage must not be confused with unbiased counterfactual policy evaluation (Li et al., 2010): https://arxiv.org/abs/1003.5956 . diff --git a/include/same/application.hpp b/include/same/application.hpp index e329e39..06b7776 100644 --- a/include/same/application.hpp +++ b/include/same/application.hpp @@ -1,7 +1,9 @@ #pragma once #include "same/config.hpp" +#include #include #include +#include namespace same { /// Explicit presentation policy; library callers retain legacy TSV by default. /// 显式展示策略;库调用方默认保留旧 TSV 格式。 @@ -44,4 +46,30 @@ int run(const std::filesystem::path& root, const Config& config, std::ostream& o */ int run(const std::filesystem::path& root, const Config& config, std::ostream& output, std::ostream& diagnostics); +/// 显式训练三个后端;只读语料,在工作区模型库持久化成功的完整文件观测。 +/// Explicitly train all three backends on read-only corpus files, persisting complete successful +/// observations to the workspace model store. No scan cache or duplicate results are changed. +/// Example: train(workspace, corpus, Config::load(workspace), std::cout, std::cerr); +int train(const std::filesystem::path& workspace, const std::filesystem::path& corpus, + const Config& config, std::ostream& output, std::ostream& diagnostics, + std::size_t max_files = 128, std::uint64_t max_bytes = 2ULL * 1024 * 1024 * 1024); +/// 训练诊断与扫描 --summary 共用展示约定;默认不改变原有调用者输出。 +/// Training diagnostics follow scan --summary presentation without changing old callers. +struct TrainReportOptions { + /// 仅显式请求时输出完整报告。 / Emit the full report only on explicit request. + bool summary{false}; + /// 使用分区与可读单位。 / Use sections and human-readable units. + bool pretty{false}; + /// 仅对终端诊断流启用颜色。 / Color only the terminal diagnostics stream. + bool color{false}; +}; +/// 使用明确展示策略训练,原有 train 重载仍保留。 / Train with explicit presentation. +int train(const std::filesystem::path& workspace, const std::filesystem::path& corpus, + const Config& config, std::ostream& output, std::ostream& diagnostics, + std::size_t max_files, std::uint64_t max_bytes, TrainReportOptions options); +/// 多目录使用同一个全局选样预算和同一个工作区模型;空列表等价于当前目录。 +/// Multiple corpora share one global sample budget and workspace model; empty means workspace. +int train(const std::filesystem::path& workspace, const std::vector& corpora, + const Config& config, std::ostream& output, std::ostream& diagnostics, + std::size_t max_files, std::uint64_t max_bytes, TrainReportOptions options); } // namespace same diff --git a/include/same/merge.hpp b/include/same/merge.hpp new file mode 100644 index 0000000..e0a1408 --- /dev/null +++ b/include/same/merge.hpp @@ -0,0 +1,29 @@ +#pragma once +#include +#include + +namespace same { +/// 合并只修改当前工作区,来源在完整成功前不删除。 +/// Merge modifies the destination; source cleanup is deferred until full success. +struct MergeOptions { + /// 递归发现指定目录下的所有真实 .same 目录。 / Recursively discover real workspaces. + bool recursive{}; + /// 成功后移除已导入的来源工作区。 / Remove imported source state after success. + bool move{}; + /// 输出本次合并的完整计数与遥测健康状况。 / Report this merge's counts and telemetry. + bool summary{}; + /// 用分区、可读单位展示报告。 / Use sectioned, human-readable reporting. + bool pretty{}; + /// 仅显式或终端支持时写 ANSI。 / Emit ANSI only under the selected terminal policy. + bool color{}; +}; +/** 将 source 下的工作区状态导入 destination;配置永远保留目的端。 + * Import source workspace state into destination, always preserving destination config. + * @code + * same::merge_workspaces(current, source, {.recursive=true, .summary=true}, std::cerr); + * @endcode + */ +int merge_workspaces(const std::filesystem::path& destination, + const std::filesystem::path& source, MergeOptions options, + std::ostream& diagnostics); +} // namespace same diff --git a/include/same/model_store.hpp b/include/same/model_store.hpp index 944c26c..93fb48b 100644 --- a/include/same/model_store.hpp +++ b/include/same/model_store.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace same { /// 设备初始化历史独立于稳定服务时间。 / Device setup history is separate from steady service time. @@ -15,6 +16,20 @@ struct SetupHistory { /// 有效初始化观测次数。 / Number of valid setup observations. std::uint64_t samples{}; }; +/// 可替换的来源贡献;revision 单调增长,attenuation 是目标内运行衰减。 +/// Replaceable source contribution; revision is monotonic and attenuation is local run decay. +struct ModelContribution { + std::string source, key; + std::uint64_t revision{}; + double attenuation{1}; + detail::OnlineModel::State state; +}; +/// 初始化历史的可替换来源贡献。 / Replaceable source setup contribution. +struct SetupContribution { + std::string source, key; + std::uint64_t revision{}; + SetupHistory history; +}; /// 独立的冷路径学习状态库;调用方持有工作区运行锁。 /// Independent cold-path learning store; caller holds the workspace run lock. /// WAL/NORMAL 可在断电时丢失最近学习,但不涉及内容缓存或正确性。 @@ -36,10 +51,22 @@ class ModelStore { /// 原子替换,失败保留旧状态;仅在所有工作线程停止后使用。 /// Atomic replacement preserves old state on failure. Use after workers stop. bool save(std::string_view key, const detail::OnlineModel::State& state); + /// 本轮只保存新观测;对已有各来源贡献分别应用运行衰减后重建聚合。 + /// Apply run decay to each source, add local observations, then rematerialize the aggregate. + bool save_delta(std::string_view key, const detail::OnlineModel::State& delta, double decay); + /// 取出当前来源账本,用于跨工作区幂等合并。 / Export provenance for idempotent merge. + std::vector contributions() const; + /// 较新 revision 替换来源,重复或较旧快照不重复计算。 / Replace only newer revisions. + bool merge_contribution(const ModelContribution& contribution); /// 不可变初始化历史快照,允许并发读取。 / Immutable setup snapshot supports concurrent reads. std::optional load_setup(std::string_view key) const; /// 空闲阶段原子写入初始化历史。 / Atomically save setup history while idle. bool save_setup(std::string_view key, const SetupHistory& history); + /// 合并本轮实际初始化观测。 / Merge this run's actual setup observations. + bool save_setup_delta(std::string_view key, const SetupHistory& delta); + /// 导出并替换初始化历史贡献。 / Export and replace setup provenance. + std::vector setup_contributions() const; + bool merge_setup_contribution(const SetupContribution& contribution); /// 构造或保存操作的结果;成功为空,load 不改变诊断。 /// Constructor/save status; empty on success, never mutated by load. std::string_view diagnostic() const noexcept; diff --git a/include/same/resources.hpp b/include/same/resources.hpp index ee321cf..f2039eb 100644 --- a/include/same/resources.hpp +++ b/include/same/resources.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -236,7 +237,15 @@ class Resources { template auto submit_hash(F&& operation, std::uint64_t bytes) -> std::future> { - return submit_impl(std::forward(operation), bytes, true); + return submit_impl(std::forward(operation), bytes, true, std::nullopt); + } + /// 显式训练使用自动模式的真实资源预算,但必须由请求后端完成,不隐式接受 CPU 回退。 + /// Train with automatic-mode budgets while requiring the requested backend; callers must + /// reject any unavailable-backend fallback before assigning a training sample. + template + auto submit_training_hash(F&& operation, std::uint64_t bytes, BackendKind backend) + -> std::future> { + return submit_impl(std::forward(operation), bytes, true, backend); } /// 曾成功启用的上下文数,支持并发读取。 / Contexts ever enabled; concurrent reads are safe. std::size_t gpu_workers() const { @@ -319,12 +328,16 @@ class Resources { /// 操作始终被调用以发布完成;初始化异常在其 execute 内传播。 /// Always invoke callbacks to publish completion; setup errors propagate within their execute. template - auto submit_impl(F&& operation, std::uint64_t bytes, bool hash) + auto submit_impl(F&& operation, std::uint64_t bytes, bool hash, + std::optional training_backend = std::nullopt) -> std::future> { using Result = std::invoke_result_t; - auto invoke = [this, op = std::forward(operation), bytes, - hash](Worker& worker) mutable -> Result { - select_backend(worker, bytes, hash); + auto invoke = [this, op = std::forward(operation), bytes, hash, + training_backend](Worker& worker) mutable -> Result { + if (training_backend) + select_backend(worker, bytes, hash, training_backend); + else + select_backend(worker, bytes, hash); if constexpr (std::is_void_v) { std::invoke(op, worker); worker.rethrow_startup(); @@ -348,16 +361,21 @@ class Resources { /// 本地成本与有界探索,不读取时钟或其他模型。 / Local costs and bounded exploration; no clock /// or peer model reads. std::pair choose_backend(Worker& worker, std::uint64_t bytes, bool hash, - const std::array& candidates); + const std::array& candidates, + std::optional training_backend); /// 取得配置与实际容量组成的上下文,不查询驱动。 / Context from config/cached capacity, no /// driver query. detail::OnlineModel::Context context(const Worker& worker, BackendKind kind, std::uint64_t bytes) const; /// 初始化时只加载对应设备的先验。 / Load only this device prior at initialization. void load_prior(Worker& worker, BackendKind kind, const Compute& compute, std::size_t block); - bool activate_cuda(Worker& worker); + bool activate_cuda(Worker& worker, bool training); /// 捕获初始化异常但不跳过用户回调。 / Capture setup exceptions without skipping the callback. void select_backend(Worker& worker, std::uint64_t bytes, bool hash) noexcept; + /// 训练专用重载保留旧三参数符号和 Worker 布局。 / Training overload preserves the + /// existing three-argument symbol and Worker layout. + void select_backend(Worker& worker, std::uint64_t bytes, bool hash, + std::optional training_backend) noexcept; /// 所属线程首次初始化并验证,失败不重新探测。 / Owner-thread one-time /// initialization/validation, without repeated probing. bool prepare_gpu(Worker& worker); @@ -375,10 +393,10 @@ class Resources { BackendPolicy policy_{BackendPolicy::cpu}; /// 单上下文、统一内存配额;原子准入失败立即继续其他后端。 /// One context and unified-memory quota; failed atomic admission never waits. - bool select_igpu(Worker& worker); - bool discover_igpu(Worker& worker); - bool admit_cold_igpu(const Worker& worker) const; - bool begin_cold(const Worker& worker, double estimate); + bool select_igpu(Worker& worker, bool training); + bool discover_igpu(Worker& worker, bool training); + bool admit_cold_igpu(const Worker& worker, bool training) const; + bool begin_cold(const Worker& worker, double estimate, bool training); void end_cold(); std::atomic cold_start_busy_{false}; std::atomic cuda_deferred_{0}; diff --git a/include/same/store.hpp b/include/same/store.hpp index 9eaa313..db2fe68 100644 --- a/include/same/store.hpp +++ b/include/same/store.hpp @@ -60,6 +60,21 @@ class Store { /// 尽力回滚活动扫描;无扫描时无操作,清理期间不抛异常。 /// Best-effort rollback of an active scan; no-op when inactive and never throws during cleanup. void rollback_scan() noexcept; + /// 外部绝对路径缓存独立于本轮 files 视图;只用完整戳命中,绝不参与跨目录结果。 + /// Absolute archive is independent of the active files view; only exact stamps may reuse it. + std::optional cached_absolute(std::string_view path); + /// 扫描事务或导入事务内保存绝对路径摘要。 / Save an absolute cache entry in scan/import. + void save_absolute(const FileRecord& record); + /// 打开独立导入事务,不推进扫描代次或清理当前结果。 / Begin import without scan pruning. + void begin_import(); + /// 原子提交本数据库导入。 / Commit this database's import atomically. + void end_import(); + /// 回滚未提交导入。 / Roll back an unfinished import. + void rollback_import() noexcept; + /// 流式遍历当前工作区记录。 / Stream current workspace records. + void visit_files(const std::function& visitor); + /// 流式遍历全局绝对路径记录。 / Stream absolute archive records. + void visit_absolute(const std::function& visitor); /// 扫描提交后访问至少两个文件的大小/摘要桶,按大小、摘要、路径排序。 /// After scan commit, visit size/digest buckets of at least two files, ordered by /// size/digest/path. diff --git a/include/same/workspace.hpp b/include/same/workspace.hpp index 4442c40..57b44b0 100644 --- a/include/same/workspace.hpp +++ b/include/same/workspace.hpp @@ -1,7 +1,22 @@ #pragma once +#include "same/compute.hpp" +#include "same/files.hpp" #include #include +#include +#include +#include namespace same { +/// 删除前逐文件验证版本与内容,不依赖文件系统时间戳分辨率。 +/// Verify both version and content before deletion, independent of timestamp resolution. +struct WorkspaceStateFile { + std::string name; + FileStamp stamp; + Digest digest; + bool operator==(const WorkspaceStateFile&) const = default; +}; +/// 不含独占打开的 run.lock。 / Excludes the exclusively opened run.lock. +using WorkspaceStateSnapshot = std::vector; /// 创建缺失的标准配置与忽略文件,永不覆盖已有文件。 /// Create missing standard configuration and ignore files without overwriting existing files. void initialize_workspace(const std::filesystem::path& root); @@ -12,4 +27,10 @@ void initialize_workspace(const std::filesystem::path& root); /// 不支持与不遵守生命周期锁协议的旧版本并发。 /// Concurrent older versions lacking the lifecycle lock protocol are unsupported. std::size_t clean_workspace(const std::filesystem::path& root, bool recursive); +/// 在调用方持有源运行锁时采集完整直系子项快照。 / Snapshot direct state children under lock. +WorkspaceStateSnapshot snapshot_workspace_state(const std::filesystem::path& root); +/// 仅在状态与合并后的快照完全相同时删除,防止移动期间丢掉新写入。 +/// Remove state only when unchanged since merge, preventing deletion of new source work. +void clean_workspace_if_unchanged(const std::filesystem::path& root, + const WorkspaceStateSnapshot& expected); } // namespace same diff --git a/src/application.cpp b/src/application.cpp index ebee90a..2c954fb 100644 --- a/src/application.cpp +++ b/src/application.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,8 @@ #include #include #include +#include +#include namespace same { namespace { @@ -43,7 +46,7 @@ std::string path_key(const fs::path& path) { } /// Decode an internal root-relative database key without locale conversion. /// 无需本地编码转换,将内部相对路径键还原为本机路径。 -fs::path native_path(const fs::path& root, const std::string& key) { +fs::path native_path(const fs::path& root, std::string_view key) { return root / fs::path(std::u8string_view(reinterpret_cast(key.data()), key.size())); } @@ -245,8 +248,8 @@ struct Learning { /// export. std::map setup_snapshots; }; -/// 每个真实初始化形成一个观测,键相同的历史只加载一次。 -/// Each actual initialization adds one observation; load a keyed history only once. +/// 每个真实初始化形成本轮增量,历史聚合交由来源账本处理。 +/// Each actual initialization forms a run delta; the provenance ledger owns history aggregation. void add_setup_history(std::map& histories, Learning& learning, const Config& config, BackendKind backend, const DeviceProfile& device, double milliseconds) { @@ -254,8 +257,7 @@ void add_setup_history(std::map& histories, Learning& return; const auto key = learning_key(config, learning.cpu, backend, device); auto [entry, inserted] = histories.try_emplace(key); - if (inserted) - entry->second = learning.store->load_setup(key).value_or(SetupHistory{}); + static_cast(inserted); auto& history = entry->second; if (history.samples == std::numeric_limits::max()) return; @@ -264,8 +266,8 @@ void add_setup_history(std::map& histories, Learning& history.mean_ms += (milliseconds - history.mean_ms) / static_cast(history.samples); history.max_ms = std::max(history.max_ms, milliseconds); } -/// 相同键只合并一次先验,再添加每线程本轮增量;失败扫描不调用。 -/// Merge each keyed prior once, then per-worker run deltas; never called for failed scans. +/// 仅合并本轮增量;各来源历史由存储层衰减并重建,失败扫描不调用。 +/// Merge run-only deltas; storage decays/rematerializes provenance, never on failed scans. void save_learning(Learning& learning, const Config& config, const Resources& resources) { if (!learning.store) return; @@ -282,10 +284,9 @@ void save_learning(Learning& learning, const Config& config, const Resources& re continue; const auto key = learning_key(config, learning.cpu, static_cast(i), worker.devices[i]); - detail::OnlineModel::State prior{}, delta{}; - prior[i] = worker.prior[i]; + detail::OnlineModel::State delta{}; delta[i] = worker.delta[i]; - auto entry = merged.try_emplace(key, prior).first; + auto entry = merged.try_emplace(key, detail::OnlineModel::State{}).first; if (!detail::OnlineModel::merge(entry->second, delta)) throw std::runtime_error("model aggregate rejected"); } @@ -294,16 +295,16 @@ void save_learning(Learning& learning, const Config& config, const Resources& re add_setup_history(setups, learning, config, BackendKind::igpu, resources.igpu_profile(), resources.igpu_setup_ms()); for (const auto& [key, history] : setups) { - if (learning.store->save_setup(key, history)) { + if (learning.store->save_setup_delta(key, history)) { ++learning.setup_saved; - learning.setup_snapshots.emplace(key, history); + learning.setup_snapshots.emplace(key, *learning.store->load_setup(key)); } else { ++learning.save_errors; learning.diagnostic = learning.store->diagnostic(); } } for (const auto& [key, state] : merged) { - if (learning.store->save(key, state)) + if (learning.store->save_delta(key, state, learning_decay)) ++learning.saved; else { ++learning.save_errors; @@ -498,10 +499,17 @@ struct HashResult { void scan(const fs::path& root, const Config& config, Store& store, Resources& resources, Counters& counters, bool recursive, telemetry::Telemetry* trace = nullptr, Clock::time_point trace_start = {}) { + const auto cache_root = fs::canonical(root); + const auto absolute_key = [&](std::string_view relative) { + return path_key(native_path(cache_root, relative)); + }; detail::CompletionJobs pending(resources, config.queue_capacity); auto save = [&](const FileRecord& record) { const auto start = Clock::now(); store.save(record); + auto absolute = record; + absolute.path = absolute_key(record.path); + store.save_absolute(absolute); counters.database_work_ms += milliseconds(start, Clock::now()); }; auto drain = [&] { @@ -611,8 +619,15 @@ void scan(const fs::path& root, const Config& config, Store& store, Resources& r std::move(entry->reader)}; counters.scanned_bytes += candidate.stamp.size; const auto queried = Clock::now(); - const bool cached = - !config.rehash && store.mark_if_unchanged(candidate.path, candidate.stamp); + bool cached = !config.rehash && store.mark_if_unchanged(candidate.path, candidate.stamp); + if (!cached && !config.rehash) { + auto archived = store.cached_absolute(absolute_key(candidate.path)); + if (archived && archived->stamp == candidate.stamp) { + archived->path = candidate.path; + store.save(*archived); + cached = true; + } + } counters.database_work_ms += milliseconds(queried, Clock::now()); if (cached) { ++counters.cached; @@ -631,9 +646,8 @@ void scan(const fs::path& root, const Config& config, Store& store, Resources& r counters.walk_task_peak = stats.task_peak; counters.walk_result_peak = stats.result_peak; const auto committed = Clock::now(); - // 淘汰未访问记录,也隔离从递归扫描切换到浅扫描后留下的子目录缓存。 - // Prune unseen records, including child-directory cache entries after switching to shallow - // mode. + // 仅清理本轮结果视图;绝对路径归档跨工作区保留,但不参与本轮候选比较。 + // Prune only the active result view; the absolute archive persists but is not a candidate. store.end_scan(); counters.database_work_ms += milliseconds(committed, Clock::now()); } @@ -1428,6 +1442,9 @@ void render_telemetry(const telemetry::Telemetry* trace, const telemetry::Stats& stats.errors ? ReportTone::danger : stats.dropped ? ReportTone::warning : ReportTone::good); + if (stats.dropped) + report.row("Trace loss", "diagnostic events omitted; inspect aggregate metrics", + ReportTone::warning); report.row("Trace pressure", std::to_string(stats.queue_high_water) + " queued peak | " + std::to_string(stats.truncated) + " truncated", @@ -1722,4 +1739,648 @@ int run(const fs::path& root, const Config& config, std::ostream& output, std::o std::rethrow_exception(failure); return 0; } + +namespace { +/// 训练语料只保存路径和元数据,不保留遍历期打开的文件句柄。 +/// Retain paths and stamps, not metadata-stage file handles, while choosing a bounded corpus. +struct TrainFile { + fs::path root; + std::string path; + FileStamp stamp; + std::uint64_t rank{}; +}; + +/// 稳定路径次序避免元数据线程完成顺序决定训练集。 / Stable path ranking prevents metadata +/// completion order from choosing the training corpus. +std::uint64_t train_rank(std::string_view root, std::string_view path) noexcept { + std::uint64_t value = 14695981039346656037ULL; + for (unsigned char byte : root) { + value ^= byte; + value *= 1099511628211ULL; + } + value ^= '/'; + value *= 1099511628211ULL; + for (unsigned char byte : path) { + value ^= byte; + value *= 1099511628211ULL; + } + return value; +} + +/// 拒绝重叠语料,避免同一个文件被多次选入并强化同一观测。 +/// Reject overlapping roots rather than counting one file repeatedly. +std::vector training_roots(const fs::path& workspace, + const std::vector& corpora) { + std::vector roots; + for (const auto& input : corpora.empty() ? std::vector{workspace} : corpora) { + const auto absolute = fs::absolute(input).lexically_normal(); + if (!fs::is_directory(absolute) || is_reparse_point(absolute)) + throw std::runtime_error("training corpus must be a real directory"); + const auto root = fs::canonical(absolute); + for (const auto& existing : roots) { + const auto from_existing = root.lexically_relative(existing); + const auto from_root = existing.lexically_relative(root); + const auto within = [](const fs::path& relative) { + return !relative.empty() && *relative.begin() != ".."; + }; + if (within(from_existing) || within(from_root)) + throw std::runtime_error("training corpus directories overlap"); + } + roots.push_back(root); + } + return roots; +} + +/// 按大小区间有界抽样,随后交错选取;不读取或修改文件内容。 +/// Select bounded size-stratified metadata before reading any file contents. +std::vector training_files(const std::vector& corpora, const Config& config, + std::size_t max_files, std::uint64_t max_bytes, + std::uint64_t& discovered) { + if (!max_files || max_files > 4096 || !max_bytes) + throw std::invalid_argument("train limits require 1..4096 files and positive bytes"); + constexpr std::size_t bands = detail::OnlineModel::band_count; + std::array, bands> choices; + const auto per_band = std::min(max_files, 64); + for (const auto& corpus : corpora) { + const auto root_key = path_key(corpus); + ParallelWalk walk(corpus, config.metadata_workers, config.queue_capacity, true); + while (auto entry = walk.next()) { + ++discovered; + if (!entry->stamp.size || entry->stamp.size > max_bytes) + continue; + const auto band = + (std::bit_width(entry->stamp.size) - 1) / detail::OnlineModel::band_shift; + TrainFile file{corpus, std::move(entry->path), std::move(entry->stamp), 0}; + file.rank = train_rank(root_key, file.path); + auto& group = choices[band]; + if (group.size() < per_band) + group.push_back(std::move(file)); + else { + const auto worst = std::max_element( + group.begin(), group.end(), [](const TrainFile& a, const TrainFile& b) { + return std::tie(a.rank, a.root, a.path) < std::tie(b.rank, b.root, b.path); + }); + if (std::tie(file.rank, file.root, file.path) < + std::tie(worst->rank, worst->root, worst->path)) + *worst = std::move(file); + } + } + } + for (auto& group : choices) + std::sort(group.begin(), group.end(), [](const TrainFile& a, const TrainFile& b) { + return std::tie(a.rank, a.root, a.path) < std::tie(b.rank, b.root, b.path); + }); + std::vector selected; + selected.reserve(std::min(max_files, bands * per_band)); + std::uint64_t total = 0; + for (std::size_t round = 0; round < per_band && selected.size() < max_files; ++round) { + for (auto& group : choices) { + if (selected.size() == max_files) + break; + if (round >= group.size() || group[round].stamp.size > max_bytes - total) + continue; + total += group[round].stamp.size; + selected.push_back(std::move(group[round])); + } + } + return selected; +} + +/// 训练复用扫描遥测库,但终态指标与阶段跨度走不丢失的结束槽。 +/// Training reuses the scan telemetry store; terminal metrics/stages bypass the event queue. +struct TrainTrace { + Clock::time_point start{Clock::now()}; + std::size_t corpora_count{}; + std::unique_ptr writer; + telemetry::FinalRecord final; + std::array stage_ms{}; + bool incomplete{}; + + /// 只记录工作区与语料身份,不记录文件内容。 / Record workspace/corpus identities, never bytes. + TrainTrace(const fs::path& workspace, const std::vector& corpora, + const Config& config, std::size_t max_files, std::uint64_t max_bytes, + std::ostream& diagnostics) + : corpora_count(corpora.size()) { + if (!config.telemetry) + return; + try { + std::ostringstream settings; + settings << "{\"effective\":" << telemetry_config(config, {}); + if (corpora.size() == 1) { + settings << ",\"corpus\":"; + quoted_path(settings, path_key(corpora.front())); + } + settings << ",\"corpora\":["; + for (std::size_t i = 0; i < corpora.size(); ++i) { + if (i) + settings << ','; + quoted_path(settings, path_key(corpora[i])); + } + settings << ']'; + settings << ",\"max_files\":" << max_files << ",\"max_bytes\":" << max_bytes << '}'; + telemetry::RunInfo info; + info.version = SAME_VERSION; + info.command = "train"; + info.root = path_key(workspace); + info.config_json = settings.str(); + telemetry::Options options; + options.queue_capacity = config.telemetry_queue_capacity; + options.event_cap = config.telemetry_max_events; + options.retain_runs = config.telemetry_retention_runs; + writer = std::make_unique(workspace / ".same" / "telemetry.db", + std::move(info), options); + } catch (...) { + diagnostics << "Telemetry warning: cannot allocate training run metadata.\n"; + } + } + + /// 固定五个阶段记录在终态槽,低内存错误另行提示。 / Fixed stages use the final slot. + void stage(unsigned slot, Clock::time_point begin, Clock::time_point end, + bool success) noexcept { + stage_ms[slot] = milliseconds(begin, end); + if (!writer) + return; + constexpr std::array names{ + "train.select", "train.initialize", "train.measure", "train.persist", "train.output"}; + telemetry::Event event; + event.type = "span"; + event.name = names[slot]; + event.severity = success ? "info" : "error"; + event.span_id = slot + 2; + event.parent_span_id = 1; + event.time_ns = static_cast( + std::chrono::duration_cast(begin - start).count()); + event.duration_ns = static_cast( + std::chrono::duration_cast(end - begin).count()); + event.value = success ? 1 : 0; + try { + final.events.push_back(event); + } catch (...) { + incomplete = true; + } + } + + /// 每个后端尝试有独立跨度,丢弃由写入器显式计数。 + /// One span per backend attempt; queue loss remains visible in writer counters. + void attempt(BackendKind backend, std::size_t worker, std::uint64_t file_index, + std::uint64_t bytes, Clock::time_point begin, bool success) noexcept { + if (!writer) + return; + const auto end = Clock::now(); + telemetry::Event event; + event.type = "span"; + event.name = "train.backend_attempt"; + event.severity = success ? "info" : "error"; + event.backend = backend_name(backend); + event.worker = static_cast(worker); + event.span_id = 1000000 + file_index * 3 + static_cast(backend); + event.parent_span_id = 1; + event.time_ns = static_cast( + std::chrono::duration_cast(begin - start).count()); + event.duration_ns = static_cast( + std::chrono::duration_cast(end - begin).count()); + event.bytes = bytes; + event.value = success ? 1 : 0; + writer->emit(event); + } +}; + +/// 作用域退出自动记录失败阶段;不在异常路径遗漏阶段状态。 +/// Scope exit records failed stages during exception unwinding. +struct TrainStage { + TrainTrace& trace; + unsigned slot; + Clock::time_point begin{Clock::now()}; + bool complete{}; + ~TrainStage() { + trace.stage(slot, begin, Clock::now(), complete); + } +}; + +/// 与普通扫描相同的完整文件服务标签;设备失败不得伪装成 CPU 回退观测。 +/// Full-file service label matching scan semantics; a device failure must not become a CPU label. +Digest train_digest(Worker& worker, const TrainFile& file, BackendKind requested) { + worker.rethrow_startup(); + if (worker.decision_backend != requested || worker.compute->kind() != requested) + throw std::runtime_error(std::string("training backend unavailable: ") + + backend_name(requested)); + ++(requested == BackendKind::cpu ? worker.cpu_hashes + : requested == BackendKind::cuda ? worker.gpu_hashes + : worker.igpu_hashes); + const auto path = native_path(file.root, file.path); + FileReader reader(path); + // 扫描通常复用元数据阶段已打开的句柄;单独训练在计时前重开,保持服务标签口径。 + // Scan usually reuses a metadata-stage handle; reopen before timing to match its service label. + const auto start = Clock::now(); + unchanged(path, reader, file.stamp); + auto hasher = worker.compute->hasher(); + std::uint64_t total = 0; + for (;;) { + const auto count = reader.read(worker.first); + if (!count) + break; + if (count > file.stamp.size - total) + throw std::runtime_error("file grew during training: " + file.path); + hasher->update(std::span(worker.first).first(count)); + total += count; + worker.hash_bytes += count; + (requested == BackendKind::cpu ? worker.cpu_hash_bytes + : requested == BackendKind::cuda ? worker.gpu_hash_bytes + : worker.igpu_hash_bytes) += count; + } + if (total != file.stamp.size) + throw std::runtime_error("file shrank during training: " + file.path); + const auto digest = hasher->finish(); + unchanged(path, reader, file.stamp); + const auto elapsed = milliseconds(start, Clock::now()); + worker.sample = {file.stamp.size, elapsed, requested == BackendKind::cuda, true, requested}; + return digest; +} + +/// 同一文件交错提交三后端,待单实例核显彻底归还后核对完整摘要。 +/// Submit all three backends for one file and wait for iGPU handback before digest comparison. +void train_one_file(Resources& resources, TrainTrace& trace, const TrainFile& file, + std::size_t index, std::array& submitted) { + constexpr std::array backends{BackendKind::cpu, BackendKind::cuda, + BackendKind::igpu}; + std::array, 3> pending; + for (unsigned offset = 0; offset < 3; ++offset) { + const auto backend = backends[(index + offset) % 3]; + const auto slot = static_cast(backend); + pending[slot] = resources.submit_training_hash( + [&, backend, index](Worker& worker) { + const auto began = Clock::now(); + try { + auto digest = train_digest(worker, file, backend); + trace.attempt(backend, worker.index, index, file.stamp.size, began, true); + return digest; + } catch (...) { + trace.attempt(backend, worker.index, index, file.stamp.size, began, false); + throw; + } + }, + file.stamp.size, backend); + ++submitted[slot]; + } + std::array digests; + for (unsigned slot = 0; slot < 3; ++slot) + digests[slot] = pending[slot].get(); + // future 可在池释放单实例 iGPU 前就绪;下一文件等待实际任务收尾。 + // A future may be ready before the pool releases its single iGPU instance. + resources.wait_idle(); + if (digests[0] != digests[1] || digests[0] != digests[2]) + throw std::runtime_error("training digest mismatch: " + file.path); +} + +/// 使用与自动扫描完全相同的设备身份和预算加载先验。 +/// Load priors with the same device identities and budgets as automatic scanning. +std::unique_ptr open_training_resources(const fs::path& workspace, + const Config& training, Learning& learning) { + const auto startup = Clock::now(); + learning.cpu = make_cpu_compute()->profile(); + if (learning.cpu.device_name.empty()) + throw std::runtime_error("CPU identity unavailable; cannot persist training"); + learning.store = std::make_unique(workspace / ".same" / "model.db"); + if (!learning.store->diagnostic().empty()) + throw std::runtime_error("model store unavailable: " + + std::string(learning.store->diagnostic())); + learning.startup_ms = milliseconds(startup, Clock::now()); + detail::ModelPriorLoader loader = [&](BackendKind backend, const DeviceProfile& device) { + ++learning.loads; + const auto prior = + learning.store->load(learning_key(training, learning.cpu, backend, device)); + if (!prior) + return detail::OnlineModel::State{}; + detail::OnlineModel model; + if (!model.initialize(*prior, learning_decay)) + throw std::runtime_error("stored training prior is invalid"); + ++learning.hits; + return model.prior(); + }; + detail::SetupPriorLoader setup_loader = [&](BackendKind backend, const DeviceProfile& device) { + ++learning.setup_loads; + const auto history = + learning.store->load_setup(learning_key(training, learning.cpu, backend, device)); + if (history) + ++learning.setup_hits; + return history ? history->max_ms : 0.0; + }; + return std::make_unique(training, std::move(loader), std::move(setup_loader)); +} + +/// 终态快照保存失败时也能解释已经做了多少工作。 +/// Final state explains partial work even if training fails before persistence. +struct TrainFacts { + std::uint64_t discovered{}, selected{}, useful{}, verified{}, bytes{}, read_bytes{}; + std::array submitted{}; + detail::OnlineModel::Snapshot snapshot; + double cuda_setup_ms_sum{}, igpu_discovery_ms{}, igpu_setup_ms{}; +}; + +/// 与扫描报告共用可读单位、分区和标准错误输出;样本数不是性能提升率。 +/// Share scan's units, sections and stderr convention; sample counts are not speedups. +void render_train_summary(const TrainFacts& facts, const Learning& learning, + const TrainTrace& trace, double elapsed, std::ostream& out, + TrainReportOptions options) { + if (!options.summary) + return; + const auto& model = facts.snapshot; + if (options.pretty) { + PrettyReport report(out, options.color); + report.heading("Training summary"); + report.section("Results"); + report.row("Corpora", std::to_string(trace.corpora_count) + " directories", + ReportTone::info); + report.row("Files", + std::to_string(facts.selected) + " selected | " + + std::to_string(facts.verified) + " verified | " + + std::to_string(facts.discovered) + " discovered", + ReportTone::good); + report.row("GPU-eligible", std::to_string(facts.useful) + " selected files", + facts.useful < 4 ? ReportTone::warning : ReportTone::good); + report.row("Samples", + std::to_string(model.cpu_samples) + " CPU | " + + std::to_string(model.gpu_samples) + " CUDA | " + + std::to_string(model.igpu_samples) + " iGPU", + ReportTone::accent); + report.section("Storage and I/O"); + report.row("Corpus data", human_bytes(static_cast(facts.bytes)) + " logical", + ReportTone::info); + report.row("Read", + human_bytes(static_cast(facts.read_bytes)) + + " logical across all backends (not physical disk I/O)", + ReportTone::info); + report.section("Performance"); + report.row("Elapsed", human_duration(elapsed) + " (excludes telemetry drain)", + ReportTone::good); + constexpr std::array labels{"Select", "Initialize", "Measure", "Persist", + "Output"}; + for (unsigned slot = 0; slot < labels.size(); ++slot) + report.row(labels[slot], human_duration(trace.stage_ms[slot]), ReportTone::info); + report.row("CUDA setup", human_duration(facts.cuda_setup_ms_sum) + " (summed workers)", + ReportTone::info); + report.row("iGPU startup", + "discover " + human_duration(facts.igpu_discovery_ms) + " | activate " + + human_duration(facts.igpu_setup_ms), + ReportTone::info); + report.section("Online routing model"); + report.row("Prediction checks", + std::to_string(model.predicted_samples) + " | absolute error sum " + + human_duration(model.absolute_error_sum_ms), + ReportTone::accent); + report.row("Out of domain", std::to_string(model.out_of_domain_samples), + model.out_of_domain_samples ? ReportTone::warning : ReportTone::muted); + report.row("Evidence", "calibration only; not held-out scan speedup validation", + ReportTone::muted); + report.row("Use", "subsequent auto scans in this workspace load the same model", + ReportTone::good); + render_learning(learning, true, out, true, options.color); + return; + } + out << "train_corpora=" << trace.corpora_count << " train_discovered_files=" << facts.discovered + << " train_selected_files=" << facts.selected << " train_verified_files=" << facts.verified + << " train_gpu_eligible_files=" << facts.useful << " train_logical_bytes=" << facts.bytes + << " train_read_bytes=" << facts.read_bytes << " train_cpu_samples=" << model.cpu_samples + << " train_cuda_samples=" << model.gpu_samples + << " train_igpu_samples=" << model.igpu_samples + << " train_predicted_samples=" << model.predicted_samples + << " train_out_of_domain_samples=" << model.out_of_domain_samples + << " train_absolute_error_sum_ms=" << model.absolute_error_sum_ms + << " train_elapsed_ms=" << elapsed; + constexpr std::array phases{"select", "initialize", "measure", "persist", + "output"}; + for (unsigned slot = 0; slot < phases.size(); ++slot) + out << " train_" << phases[slot] << "_ms=" << trace.stage_ms[slot]; + out << " train_cuda_setup_ms_sum=" << facts.cuda_setup_ms_sum + << " train_igpu_discovery_ms=" << facts.igpu_discovery_ms + << " train_igpu_setup_ms=" << facts.igpu_setup_ms << '\n'; + render_learning(learning, true, out, false, false); +} + +/// 完整终态指标与状态独立于可丢弃的逐任务事件;排空耗时只在结束后报告。 +/// Complete final metrics do not depend on queued attempts; drain time is reported afterward. +void finish_train_trace(TrainTrace& trace, const TrainFacts& facts, const Learning& learning, + Clock::time_point completed, bool failed, std::string_view error, + std::ostream& diagnostics, TrainReportOptions options) { + if (!trace.writer) { + if (options.summary) + render_telemetry(nullptr, {}, milliseconds(trace.start, Clock::now()), diagnostics, + options.pretty, options.color); + return; + } + try { + trace.final.status = failed ? "failed" : "completed"; + trace.final.error = error; + auto metric = [&](std::string name, double value, std::string unit = "count") { + trace.final.metrics.push_back({std::move(name), value, std::move(unit)}); + }; + metric("train.discovered_files", static_cast(facts.discovered)); + metric("train.selected_files", static_cast(facts.selected)); + metric("train.eligible_files", static_cast(facts.useful)); + metric("train.verified_files", static_cast(facts.verified)); + metric("train.logical_bytes", static_cast(facts.bytes), "bytes"); + metric("train.read_bytes", static_cast(facts.read_bytes), "bytes"); + metric("train.submitted_cpu", static_cast(facts.submitted[0])); + metric("train.submitted_cuda", static_cast(facts.submitted[1])); + metric("train.submitted_igpu", static_cast(facts.submitted[2])); + metric("train.cpu_samples", static_cast(facts.snapshot.cpu_samples)); + metric("train.cuda_samples", static_cast(facts.snapshot.gpu_samples)); + metric("train.igpu_samples", static_cast(facts.snapshot.igpu_samples)); + metric("train.predicted_samples", static_cast(facts.snapshot.predicted_samples)); + metric("train.absolute_error_sum_ms", facts.snapshot.absolute_error_sum_ms, "ms"); + metric("train.out_of_domain_samples", + static_cast(facts.snapshot.out_of_domain_samples)); + metric("train.cuda_setup_ms_sum", facts.cuda_setup_ms_sum, "ms"); + metric("train.igpu_discovery_ms", facts.igpu_discovery_ms, "ms"); + metric("train.igpu_setup_ms", facts.igpu_setup_ms, "ms"); + metric("train.model_load_attempts", static_cast(learning.loads.load())); + metric("train.model_prior_hits", static_cast(learning.hits.load())); + metric("train.model_saved_keys", static_cast(learning.saved)); + metric("train.model_save_errors", static_cast(learning.save_errors)); + constexpr std::array stages{"select", "initialize", "measure", "persist", + "output"}; + for (unsigned slot = 0; slot < stages.size(); ++slot) + metric("train." + std::string(stages[slot]) + "_ms", trace.stage_ms[slot], "ms"); + metric("train.elapsed_ms", milliseconds(trace.start, completed), "ms"); + trace.final.parameters.push_back( + {"train", "selection", "factor-four-band/deterministic-path-rank"}); + trace.final.parameters.push_back( + {"train", "span_id_scheme", "1000000+file_index*3+backend_index"}); + trace.final.parameters.push_back( + {"model", "feature_version", std::to_string(detail::OnlineModel::feature_version)}); + trace.final.parameters.push_back({"model", "run_decay", std::to_string(learning_decay)}); + telemetry::Event run; + run.type = "span"; + run.name = "train.run"; + run.severity = failed ? "error" : "info"; + run.span_id = 1; + run.duration_ns = static_cast( + std::chrono::duration_cast(completed - trace.start).count()); + run.value = failed ? 0 : 1; + trace.final.events.push_back(run); + telemetry::Event log; + log.type = "log"; + log.name = failed ? "train.failed" : "train.completed"; + log.severity = failed ? "error" : "info"; + log.message = error; + log.span_id = 1; + log.time_ns = run.duration_ns; + log.value = failed ? 0 : 1; + trace.final.events.push_back(log); + } catch (...) { + trace.incomplete = true; + } + const auto stats = trace.writer->finish(std::move(trace.final)); + if (options.summary) + render_telemetry(trace.writer.get(), stats, milliseconds(trace.start, Clock::now()), + diagnostics, options.pretty, options.color); + if (stats.errors) + diagnostics << "Telemetry warning: " << stats.error.c_str() << '\n'; +} +} // namespace + +int train(const fs::path& workspace, const fs::path& corpus, const Config& config, + std::ostream& output, std::ostream& diagnostics, std::size_t max_files, + std::uint64_t max_bytes) { + return train(workspace, corpus, config, output, diagnostics, max_files, max_bytes, {}); +} + +int train(const fs::path& workspace, const fs::path& corpus, const Config& config, + std::ostream& output, std::ostream& diagnostics, std::size_t max_files, + std::uint64_t max_bytes, TrainReportOptions options) { + return train(workspace, std::vector{corpus}, config, output, diagnostics, max_files, + max_bytes, options); +} + +int train(const fs::path& workspace, const std::vector& corpora, const Config& config, + std::ostream& output, std::ostream& diagnostics, std::size_t max_files, + std::uint64_t max_bytes, TrainReportOptions options) { + // 保留公共签名供既有调用者链接;训练无结果记录,只有显式 summary 才展示统计。 + // Preserve the public signature; train has no result records, only opt-in summary metrics. + static_cast(output); + Config training = config; + training.backend = "auto"; + training.pgo = true; + training.validate(); + if (!max_files || max_files > 4096 || !max_bytes) + throw std::invalid_argument("train limits require 1..4096 files and positive bytes"); + const auto roots = training_roots(workspace, corpora); + WorkspaceLock workspace_lock(workspace); + prepare_state(workspace); + RunLock lock(workspace / ".same" / "run.lock"); + TrainTrace trace(workspace, roots, training, max_files, max_bytes, diagnostics); + std::vector files; + Learning learning; + std::unique_ptr resources; + std::uint64_t discovered = 0, useful = 0, verified = 0; + std::uint64_t bytes = 0; + std::array submitted{}; + std::exception_ptr failure; + std::string failure_text; + try { + { + TrainStage stage{trace, 0}; + files = training_files(roots, training, max_files, max_bytes, discovered); + if (files.empty()) + throw std::runtime_error("training corpus has no nonempty eligible regular files"); + useful = static_cast( + std::count_if(files.begin(), files.end(), [&](const TrainFile& file) { + return file.stamp.size >= training.gpu_min_bytes; + })); + if (!useful) + throw std::runtime_error("training corpus has no file at or above gpu_min_bytes"); + stage.complete = true; + } + { + TrainStage stage{trace, 1}; + resources = open_training_resources(workspace, training, learning); + stage.complete = true; + } + { + TrainStage stage{trace, 2}; + for (std::size_t index = 0; index < files.size(); ++index) { + const auto& file = files[index]; + train_one_file(*resources, trace, file, index, submitted); + ++verified; + bytes += file.stamp.size; + } + resources->wait_idle(); + const auto snapshot = resources->profile_snapshot(); + if (snapshot.cpu_samples != files.size() || snapshot.gpu_samples != files.size() || + snapshot.igpu_samples != files.size()) + throw std::runtime_error( + "training did not produce one valid sample per backend/file"); + stage.complete = true; + } + { + TrainStage stage{trace, 3}; + save_learning(learning, training, *resources); + if (learning.save_errors || learning.saved < 3) + throw std::runtime_error("training model save failed: " + learning.diagnostic); + stage.complete = true; + } + } catch (...) { + failure = std::current_exception(); + try { + std::rethrow_exception(failure); + } catch (const std::exception& error) { + failure_text = error.what(); + } catch (...) { + failure_text = "non-standard training exception"; + } + } + if (resources) + resources->wait_idle(); + const auto snapshot = + resources ? resources->profile_snapshot() : detail::OnlineModel::Snapshot{}; + double cuda_setup_ms_sum = 0; + if (resources) + for (const auto& worker : resources->worker_profiles()) + cuda_setup_ms_sum += worker.setup_ms; + const auto igpu_discovery_ms = resources ? resources->igpu_discovery_ms() : 0.0; + const auto igpu_setup_ms = resources ? resources->igpu_setup_ms() : 0.0; + const auto read_bytes = resources ? resources->read_bytes().first : 0; + resources.reset(); + if (!failure) { + try { + TrainStage stage{trace, 4}; + if (useful < 4 || files.size() < 4) + diagnostics << "Training warning: fewer than four GPU-eligible files; " + "prediction coverage may be weak.\n"; + stage.complete = true; + } catch (...) { + failure = std::current_exception(); + try { + std::rethrow_exception(failure); + } catch (const std::exception& error) { + failure_text = error.what(); + } catch (...) { + failure_text = "non-standard training diagnostic exception"; + } + } + } + const auto completed = Clock::now(); + TrainFacts facts; + facts.discovered = discovered; + facts.selected = files.size(); + facts.useful = useful; + facts.verified = verified; + facts.bytes = bytes; + facts.read_bytes = read_bytes; + facts.submitted = submitted; + facts.snapshot = snapshot; + facts.cuda_setup_ms_sum = cuda_setup_ms_sum; + facts.igpu_discovery_ms = igpu_discovery_ms; + facts.igpu_setup_ms = igpu_setup_ms; + if (!failure) + render_train_summary(facts, learning, trace, milliseconds(trace.start, completed), + diagnostics, options); + finish_train_trace(trace, facts, learning, completed, bool(failure), failure_text, diagnostics, + options); + if (trace.incomplete) + diagnostics << "Telemetry warning: training final metadata is incomplete.\n"; + if (failure) + std::rethrow_exception(failure); + return 0; +} } // namespace same diff --git a/src/main.cpp b/src/main.cpp index 09b1459..580afe4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,38 +1,81 @@ #include "same/application.hpp" +#include "same/merge.hpp" #include "same/terminal.hpp" #include "same/workspace.hpp" +#include #include +#include #include namespace { -/// Print the command contract without touching workspace state. -/// 输出命令契约,不访问或修改工作区状态。 -void help() { - std::cout - << "same " SAME_VERSION " - exact duplicate files in the working directory\n" - "Usage: same [scan] [-r] [--summary] [scan options]\n" - " same new\n" - " same clean [-r]\n" - " scan (default) scan direct files; -r, --recursive includes subdirectories\n" - " new create complete defaults and recommended .same/ignore; preserve " - "existing files\n" - " clean remove current .same; -r also removes descendant .same directories\n" - "Scan options:\n" - " --summary show the full current-run diagnostic report (off by default)\n" - " --rehash ignore cached hashes for this scan\n" - " --cpu force the CPU backend\n" - " --cuda select CUDA (CPU fallback when unavailable)\n" - " --igpu select integrated OpenCL GPU (CPU fallback when unavailable/busy)\n" - " --no-pgo disable runtime profile-guided routing (not compiler PGO)\n" - " --no-telemetry disable persistent local diagnostics (not PGO or summary)\n" - " --unique-files show unmatched paths (TSV: group 0)\n" - " --color=auto|always|never auto honors NO_COLOR and redirection\n" - " --format=auto|pretty|tsv auto uses pretty on terminals, TSV otherwise\n" - " -h, --help show help\n" - " --version show version\n" - "Configuration: .same/config.toml; Git-compatible patterns: .same/ignore\n" - "No directory argument: all commands operate on the working directory.\n" - "Results: stdout; requested profiling and warnings/errors: stderr.\n"; +/// 使用已有终端颜色策略展示层级;纯文本保留相同标签和内容。 +/// Render hierarchy with the existing terminal policy; plain text retains labels and content. +void help(same::ColorMode mode) { + same::TerminalColor terminal(mode); + const bool color = terminal.enabled(); + auto styled = [&](std::string_view value, std::string_view sgr) { + if (color) + std::cout << sgr; + std::cout << value; + if (color) + std::cout << "\033[0m"; + }; + auto heading = [&](std::string_view title) { + std::cout << '\n'; + styled(title, "\033[1;36m"); + std::cout << '\n'; + }; + auto row = [&](std::string_view name, std::string_view description) { + std::cout << " "; + styled(name, "\033[1;32m"); + if (name.size() < 28) + std::cout << std::string(28 - name.size(), ' '); + else + std::cout << ' '; + std::cout << description << '\n'; + }; + styled("same " SAME_VERSION, "\033[1;35m"); + std::cout << " - exact duplicate files in the working directory\n"; + heading("Usage"); + std::cout << " same [scan] [-r] [--summary] [scan options]\n" + " same train [dirs...] [--max-files=N] [--max-bytes=N] [--summary]\n" + " same merge [dir] [-r] [--move] [--summary]\n" + " same new\n" + " same clean [-r]\n"; + heading("Commands"); + row("scan (default)", "scan direct files; -r includes subdirectories"); + row("train", "calibrate the model used by auto scans in this workspace"); + row("merge", "import workspace state, preserving local configuration"); + row("new", "create defaults and .same/ignore; preserve existing files"); + row("clean", "remove current .same; -r also removes descendant .same directories"); + heading("Scan options"); + row("--rehash", "ignore cached hashes for this scan"); + row("--cpu", "force the CPU backend"); + row("--cuda", "select CUDA (CPU fallback when unavailable)"); + row("--igpu", "select integrated OpenCL GPU (CPU fallback when unavailable/busy)"); + row("--no-pgo", "disable runtime profile-guided routing"); + row("--unique-files", "show unmatched paths (TSV: group 0)"); + heading("Train options"); + row("dirs...", "read-only training directories (default: working directory)"); + row("--corpus=DIR", "compatible alias for a training directory"); + row("--max-files=N", "selected-file ceiling (default: 128; maximum: 4096)"); + row("--max-bytes=N", "logical-byte ceiling (default: 2 GiB)"); + heading("Merge options"); + row("dir", "source directory (default: working directory)"); + row("-r, --recursive", "discover descendant .same workspaces"); + row("--move", "remove source workspaces only after successful import"); + heading("Shared options"); + row("--summary", "show the full current-run diagnostic report"); + row("--no-telemetry", "disable local diagnostics, not model learning"); + row("--color=auto|always|never", "auto honors NO_COLOR and redirection"); + row("--format=auto|pretty|tsv", "auto uses pretty on terminals, TSV otherwise"); + row("-h, --help", "show help"); + row("--version", "show version"); + heading("Notes"); + std::cout << " Configuration: .same/config.toml; ignore rules: .same/ignore\n" + " Commands use the working directory; train saves its model there.\n" + " Scan results: stdout; summaries and warnings/errors: stderr.\n" + " Successful train is quiet unless --summary is requested.\n"; } } // namespace @@ -47,19 +90,60 @@ int main(int argc, char** argv) { if (argc > 1 && !std::string_view(argv[1]).starts_with("-")) { command = argv[1]; first = 2; - if (command != "scan" && command != "new" && command != "clean") + if (command != "scan" && command != "new" && command != "clean" && command != "train" && + command != "merge") throw std::runtime_error("unknown command: " + std::string(command)); } bool rehash = false, unique_files = false; std::string_view backend; bool recursive = false, summary = false, no_pgo = false; bool no_telemetry = false; + std::vector corpora; + std::filesystem::path merge_source; + bool merge_source_set = false, move = false; + bool positional_only = false; + std::size_t train_max_files = 128; + std::uint64_t train_max_bytes = 2ULL * 1024 * 1024 * 1024; + const auto positive = [](std::string_view value) -> std::uint64_t { + std::uint64_t number = 0; + const auto* first = value.data(); + const auto* last = first + value.size(); + const auto result = std::from_chars(first, last, number); + if (value.empty() || result.ec != std::errc{} || result.ptr != last || !number) + throw std::runtime_error("training limit must be a positive decimal integer"); + return number; + }; auto color_mode = same::ColorMode::automatic; std::string_view format = "auto"; + const auto path_argument = [](std::string_view value) { + return std::filesystem::path( + std::u8string_view(reinterpret_cast(value.data()), value.size())); + }; for (int i = first; i < argc; ++i) { const std::string_view arg = argv[i]; + if ((command == "train" || command == "merge") && arg == "--") { + positional_only = true; + continue; + } + if (command == "train" && positional_only) { + corpora.push_back(path_argument(arg)); + continue; + } + if (command == "merge" && positional_only) { + if (merge_source_set) + throw std::runtime_error("merge accepts one source directory"); + merge_source = path_argument(arg); + merge_source_set = true; + continue; + } if (arg == "--help" || arg == "-h") { - help(); + auto help_mode = color_mode; + for (int later = i + 1; later < argc; ++later) { + const std::string_view option = argv[later]; + if (option.starts_with("--color=")) + help_mode = same::parse_color(option.substr(8)); + } + help(help_mode); return 0; } if (arg == "--version") { @@ -67,9 +151,57 @@ int main(int argc, char** argv) { return 0; } if ((arg == "-r" || arg == "--recursive") && command != "new") { + if (command == "train") + throw std::runtime_error("train already reads the corpus recursively"); recursive = true; continue; } + if (command == "train") { + if (arg == "--no-telemetry") + no_telemetry = true; + else if (arg == "--summary") + summary = true; + else if (arg.starts_with("--color=")) + color_mode = same::parse_color(arg.substr(8)); + else if (arg.starts_with("--format=")) { + format = arg.substr(9); + if (format != "auto" && format != "pretty" && format != "tsv") + throw std::runtime_error("invalid output format: " + std::string(format)); + } else if (arg.starts_with("--corpus=")) { + if (arg.size() == 9) + throw std::runtime_error("train requires nonempty --corpus=DIR"); + corpora.push_back(path_argument(arg.substr(9))); + } else if (arg.starts_with("--max-files=")) { + const auto value = positive(arg.substr(12)); + if (value > 4096 || value > std::numeric_limits::max()) + throw std::runtime_error("--max-files must be in [1, 4096]"); + train_max_files = static_cast(value); + } else if (arg.starts_with("--max-bytes=")) + train_max_bytes = positive(arg.substr(12)); + else if (!arg.starts_with('-')) + corpora.push_back(path_argument(arg)); + else + throw std::runtime_error("unsupported argument for train: " + std::string(arg)); + continue; + } + if (command == "merge") { + if (arg == "--move") + move = true; + else if (arg == "--summary") + summary = true; + else if (arg.starts_with("--color=")) + color_mode = same::parse_color(arg.substr(8)); + else if (arg.starts_with("--format=")) { + format = arg.substr(9); + if (format != "auto" && format != "pretty" && format != "tsv") + throw std::runtime_error("invalid output format: " + std::string(format)); + } else if (!arg.starts_with('-') && !merge_source_set) { + merge_source = path_argument(arg); + merge_source_set = true; + } else + throw std::runtime_error("unsupported argument for merge: " + std::string(arg)); + continue; + } if (command != "scan") throw std::runtime_error("unsupported argument for " + std::string(command) + ": " + std::string(arg)); @@ -109,7 +241,27 @@ int main(int argc, char** argv) { std::cout << "Removed " << removed << " .same directories\n"; return 0; } + if (command == "merge") { + same::TerminalColor diagnostic_color(color_mode, same::TerminalStream::diagnostics); + const bool pretty_profile = + format == "pretty" || + (format == "auto" && same::is_terminal(same::TerminalStream::diagnostics)); + return same::merge_workspaces( + root, merge_source_set ? merge_source : root, + {recursive, move, summary, pretty_profile, diagnostic_color.enabled()}, std::cerr); + } auto config = same::Config::load(root); + if (command == "train") { + if (no_telemetry) + config.telemetry = false; + same::TerminalColor diagnostic_color(color_mode, same::TerminalStream::diagnostics); + const bool pretty_profile = + format == "pretty" || + (format == "auto" && same::is_terminal(same::TerminalStream::diagnostics)); + return same::train(root, corpora, config, std::cout, std::cerr, train_max_files, + train_max_bytes, + {summary, pretty_profile, diagnostic_color.enabled()}); + } if (rehash) config.rehash = true; if (!backend.empty()) diff --git a/src/merge.cpp b/src/merge.cpp new file mode 100644 index 0000000..86c96a8 --- /dev/null +++ b/src/merge.cpp @@ -0,0 +1,556 @@ +#include "same/merge.hpp" +#include "same/files.hpp" +#include "same/model_store.hpp" +#include "same/run_lock.hpp" +#include "same/store.hpp" +#include "same/terminal.hpp" +#include "same/workspace.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace same { +namespace { +namespace fs = std::filesystem; + +/// 仅供合并展示的计数;每个库各自事务化,重试必须幂等。 +/// Merge counters; each database commits separately, so retries must be idempotent. +struct Counts { + std::uint64_t workspaces{}, model_parts{}, setup_parts{}, cache_imported{}; + std::uint64_t telemetry_runs{}, telemetry_events{}, ignore_rules_added{}, removed{}; +}; + +/// UTF-8 路径的持久形式与已有 state/model 身份一致。 / Persist paths as generic UTF-8. +std::string key(const fs::path& path) { + const auto bytes = path.generic_u8string(); + return {reinterpret_cast(bytes.data()), bytes.size()}; +} + +/// 从数据库 BLOB 解码路径,禁止隐式本地代码页转换。 +/// Decode a persisted UTF-8 path without an implicit locale conversion. +fs::path path_from_key(std::string_view bytes) { + return fs::path( + std::u8string_view(reinterpret_cast(bytes.data()), bytes.size())); +} + +/// 只接受不逃离来源根的相对缓存路径。 / Reject absolute and parent-traversing source keys. +fs::path source_file(const fs::path& root, std::string_view relative) { + const auto path = path_from_key(relative); + if (path.empty() || path.is_absolute()) + throw std::runtime_error("invalid relative state cache path"); + for (const auto& component : path) + if (component == "." || component == "..") + throw std::runtime_error("traversing state cache path"); + return (root / path).lexically_normal(); +} + +/// 合并不跟随 .same 链接,也不把陌生 SQLite 文件误认为 same 状态。 +/// Never follow linked state or accept special/multiply linked database files. +bool regular_state(const fs::path& path) { + const auto status = fs::symlink_status(path); + if (!fs::exists(status)) + return false; + if (!fs::is_regular_file(status) || is_reparse_point(path) || fs::hard_link_count(path) != 1) + throw std::runtime_error("unsafe merge state file: " + key(path)); + return true; +} + +/// SQLite 主库与边车都必须是普通非链接文件,避免重定向读写。 +/// Validate the SQLite main file and sidecars before opening either connection. +bool safe_database(const fs::path& path) { + const bool exists = regular_state(path); + for (const auto* suffix : {"-wal", "-shm", "-journal"}) { + auto sidecar = path; + sidecar += suffix; + if (regular_state(sidecar) && !exists) + throw std::runtime_error("orphan SQLite sidecar in merge workspace"); + } + return exists; +} + +/// 发现真实工作区,按路径稳定排序;.same 内部不继续遍历。 +/// Discover real workspaces deterministically without traversing state or links. +std::vector sources(const fs::path& destination, const fs::path& requested, + bool recursive) { + if (!fs::is_directory(requested) || is_reparse_point(requested)) + throw std::runtime_error("merge source must be a real directory"); + const auto root = fs::canonical(requested); + std::vector result; + const auto consider = [&](const fs::path& directory) { + if (directory == destination) + return; + const auto state = directory / ".same"; + const auto status = fs::symlink_status(state); + if (!fs::exists(status)) + return; + if (!fs::is_directory(status) || is_reparse_point(state)) + throw std::runtime_error("merge refuses linked or non-directory .same"); + result.push_back(directory); + }; + consider(root); + if (recursive) { + for (fs::recursive_directory_iterator it(root), end; it != end; ++it) { + const auto& entry = *it; + if (entry.path().filename() == ".same") { + it.disable_recursion_pending(); + continue; + } + if (is_reparse_point(entry.path())) { + it.disable_recursion_pending(); + continue; + } + if (entry.is_directory()) + consider(fs::canonical(entry.path())); + } + } + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + if (result.empty()) + throw std::runtime_error("merge found no other .same workspaces"); + return result; +} + +/// 外部来源没有可推断的目标子路径;子工作区则相对目标根锚定。 +/// External roots have no target subpath; descendant roots anchor at their relative location. +std::string scoped_rule(std::string line, const fs::path& destination, const fs::path& source, + bool& external) { + const auto relative = source.lexically_relative(destination); + if (relative.empty() || *relative.begin() == "..") { + external = true; + return line; + } + std::string prefix; + for (unsigned char byte : key(relative)) { + if (byte < 32 || byte == 127) + throw std::runtime_error("cannot scope ignore through control-character path"); + if (byte == '*' || byte == '?' || byte == '[' || byte == ']' || byte == '\\') + prefix += '\\'; + prefix += static_cast(byte); + } + const bool negate = line.starts_with('!'); + if (negate) + line.erase(0, 1); + const bool anchored = line.starts_with('/'); + if (anchored) + line.erase(0, 1); + auto pattern = line; + while (!pattern.empty() && pattern.back() == ' ') + pattern.pop_back(); + if (pattern.ends_with('/')) + pattern.pop_back(); + const bool anywhere = !anchored && pattern.find('/') == std::string::npos; + return (negate ? "!" : "") + std::string("/") + prefix + "/" + (anywhere ? "**/" : "") + line; +} + +/// 按字节读取至现有 Ignore 的 1 MiB 限制。 / Read within Ignore's existing 1 MiB ceiling. +std::string ignore_text(const fs::path& path) { + if (!regular_state(path)) + return {}; + if (fs::file_size(path) > 1024 * 1024) + throw std::runtime_error("ignore file exceeds 1 MiB"); + std::ifstream in(path, std::ios::binary); + const std::string content(std::istreambuf_iterator{in}, {}); + if (!in.eof() && !in) + throw std::runtime_error("cannot read merge ignore file"); + return content; +} + +/// 仅比较实际规则行,不把注释当规则。 / Compare active rule lines, not comments. +std::vector ignore_rules(const std::string& content) { + std::vector rules; + std::istringstream lines(content); + std::string line; + bool first = true; + while (std::getline(lines, line)) { + if (first && line.starts_with("\xEF\xBB\xBF")) + line.erase(0, 3); + first = false; + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (line.empty() || line.front() == '#') + continue; + rules.push_back(line); + } + return rules; +} + +/// 先写同目录临时文件再替换;失败时保持旧规则。 / Replace from a sibling temp file. +void replace_ignore(const fs::path& target, const std::string& content) { + auto temporary = target; + temporary += + ".merge-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + if (fs::exists(fs::symlink_status(temporary))) + throw std::runtime_error("ignore merge temporary path collision"); + try { + { + std::ofstream out(temporary, std::ios::binary | std::ios::trunc); + out.write(content.data(), static_cast(content.size())); + out.close(); + if (!out) + throw std::runtime_error("cannot write merged ignore rules"); + } + fs::rename(temporary, target); + } catch (...) { + std::error_code ignored; + fs::remove(temporary, ignored); + throw; + } +} + +/// 现有配置不动;规则逐条去重,子目录规则加作用域。 +/// Preserve configuration and append only missing rules, scoping descendant patterns. +void merge_ignore(const fs::path& destination, const fs::path& source, Counts& counts, + std::ostream& diagnostics) { + const auto target = destination / ".same" / "ignore"; + const auto origin = source / ".same" / "ignore"; + if (!regular_state(origin)) + return; + auto merged = ignore_text(target); + auto existing = ignore_rules(merged); + std::unordered_set known(existing.begin(), existing.end()); + bool external = false; + std::uint64_t added = 0; + for (auto& rule : ignore_rules(ignore_text(origin))) { + auto scoped = scoped_rule(std::move(rule), destination, source, external); + if (!known.insert(scoped).second) + continue; + if (!merged.empty() && merged.back() != '\n') + merged += '\n'; + merged += scoped + '\n'; + ++added; + } + if (!added) + return; + if (merged.size() > 1024 * 1024 || existing.size() + added > 4096) + throw std::runtime_error("merged ignore exceeds supported limits"); + replace_ignore(target, merged); + counts.ignore_rules_added += added; + if (external) + diagnostics << "Merge warning: external ignore rules now apply at the destination root; " + "review .same/ignore.\n"; +} + +/// 归档可包含暂时离线的文件;真正复用时仍须完整文件戳匹配。 +/// Archive even temporarily absent files; reuse still requires an exact current stamp. +void merge_state(const fs::path& destination, const fs::path& source, Counts& counts) { + const auto origin = source / ".same" / "state.db"; + if (!safe_database(origin)) + return; + safe_database(destination / ".same" / "state.db"); + Store input(origin); + Store output(destination / ".same" / "state.db"); + output.begin_import(); + try { + const auto import = [&](FileRecord record, const fs::path& path) { + record.path = key(path.lexically_normal()); + output.save_absolute(record); + ++counts.cache_imported; + }; + input.visit_files( + [&](const FileRecord& record) { import(record, source_file(source, record.path)); }); + input.visit_absolute([&](const FileRecord& record) { + const auto path = path_from_key(record.path); + if (!path.is_absolute()) + throw std::runtime_error("non-absolute archive cache path"); + import(record, path); + }); + output.end_import(); + } catch (...) { + output.rollback_import(); + throw; + } +} + +/// 来源账本替换而非整库相加;旧来源增量因此不会重复计数。 +/// Replace provenance contributions, never add opaque cumulative aggregates twice. +void merge_model(const fs::path& destination, const fs::path& source, Counts& counts) { + const auto origin = source / ".same" / "model.db"; + if (!safe_database(origin)) + return; + ModelStore input(origin); + ModelStore output(destination / ".same" / "model.db"); + if (!input.diagnostic().empty() || !output.diagnostic().empty()) + throw std::runtime_error("merge model store unavailable"); + for (const auto& part : input.contributions()) { + if (!output.merge_contribution(part)) + throw std::runtime_error("merge model contribution failed: " + + std::string(output.diagnostic())); + ++counts.model_parts; + } + for (const auto& part : input.setup_contributions()) { + if (!output.merge_setup_contribution(part)) + throw std::runtime_error("merge setup contribution failed: " + + std::string(output.diagnostic())); + ++counts.setup_parts; + } +} + +/// 独占一个短期 SQLite 连接。 / Own a short-lived SQLite connection. +struct Database { + sqlite3* value{}; + Database(const fs::path& path, int flags) { + const auto bytes = path.u8string(); + if (sqlite3_open_v2(reinterpret_cast(bytes.c_str()), &value, flags, nullptr) != + SQLITE_OK) { + sqlite3_close(value); + value = nullptr; + throw std::runtime_error("cannot open merge telemetry database"); + } + sqlite3_busy_timeout(value, 5000); + if (sqlite3_exec(value, "PRAGMA trusted_schema=OFF", nullptr, nullptr, nullptr) != + SQLITE_OK) { + sqlite3_close(value); + value = nullptr; + throw std::runtime_error("cannot configure merge telemetry database"); + } + } + ~Database() { + sqlite3_close(value); + } + Database(const Database&) = delete; + Database& operator=(const Database&) = delete; +}; + +/// 每次 SQL 错误携带当前连接原因。 / Preserve the failing SQLite connection's reason. +void sql(sqlite3* db, const char* statement) { + if (sqlite3_exec(db, statement, nullptr, nullptr, nullptr) != SQLITE_OK) + throw std::runtime_error(sqlite3_errmsg(db)); +} + +/// 语句 RAII;查询与复制期间游标始终有界。 / Bound cursors with RAII. +struct Query { + sqlite3_stmt* value{}; + Query(sqlite3* db, const char* sql) { + if (sqlite3_prepare_v2(db, sql, -1, &value, nullptr) != SQLITE_OK) + throw std::runtime_error(sqlite3_errmsg(db)); + } + ~Query() { + sqlite3_finalize(value); + } + Query(const Query&) = delete; + Query& operator=(const Query&) = delete; +}; + +/// schema 与所有权在任何写入之前检查。 / Check schema and ownership before writes. +void verify_telemetry(sqlite3* db, bool empty_allowed) { + Query version(db, "PRAGMA user_version"); + Query owner(db, "PRAGMA application_id"); + if (sqlite3_step(version.value) != SQLITE_ROW || sqlite3_step(owner.value) != SQLITE_ROW) + throw std::runtime_error("cannot read telemetry ownership"); + const auto v = sqlite3_column_int(version.value, 0); + const auto app = sqlite3_column_int(owner.value, 0); + if (empty_allowed && v == 0 && app == 0) { + Query existing(db, "SELECT count(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'"); + if (sqlite3_step(existing.value) == SQLITE_ROW && + sqlite3_column_int(existing.value, 0) == 0) + return; + } + if (!empty_allowed && v == 1 && app == 1396788564) + return; + throw std::runtime_error("unsupported or foreign telemetry database"); +} + +/// 通用定列复制,列值类型与 BLOB 精度原样保留。 / Copy typed SQLite values exactly. +std::uint64_t copy_rows(sqlite3* source, sqlite3* target, const char* select_sql, + const char* insert_sql, std::string_view run_id) { + Query rows(source, select_sql); + Query insert(target, insert_sql); + if (sqlite3_bind_text(rows.value, 1, run_id.data(), static_cast(run_id.size()), + SQLITE_TRANSIENT) != SQLITE_OK) + throw std::runtime_error("cannot bind telemetry run ID"); + std::uint64_t copied = 0; + for (;;) { + const int rc = sqlite3_step(rows.value); + if (rc == SQLITE_DONE) + break; + if (rc != SQLITE_ROW) + throw std::runtime_error(sqlite3_errmsg(source)); + sqlite3_reset(insert.value); + sqlite3_clear_bindings(insert.value); + const auto columns = sqlite3_column_count(rows.value); + for (int i = 0; i < columns; ++i) + if (sqlite3_bind_value(insert.value, i + 1, sqlite3_column_value(rows.value, i)) != + SQLITE_OK) + throw std::runtime_error(sqlite3_errmsg(target)); + if (sqlite3_step(insert.value) != SQLITE_DONE) + throw std::runtime_error(sqlite3_errmsg(target)); + ++copied; + } + return copied; +} + +/// 来源运行不可变,按 run_id 去重;复制四表并保留原事件时序。 +/// Deduplicate immutable runs by ID and copy all four related tables transactionally. +void merge_telemetry(const fs::path& destination, const fs::path& source, Counts& counts) { + const auto origin = source / ".same" / "telemetry.db"; + if (!safe_database(origin)) + return; + const auto target = destination / ".same" / "telemetry.db"; + const bool existed = safe_database(target); + Database input(origin, SQLITE_OPEN_READONLY); + verify_telemetry(input.value, false); + if (!existed) { + bool owned_empty_target = false; + try { + // 新目标复制失败时删除不完整库,重试不能被残留 schema 卡住。 + // Remove a failed first copy so a retry is not blocked by a partial schema. + Database output(target, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE); + verify_telemetry(output.value, true); + owned_empty_target = true; + auto* backup = sqlite3_backup_init(output.value, "main", input.value, "main"); + if (!backup) + throw std::runtime_error("cannot start telemetry copy"); + const int copied = sqlite3_backup_step(backup, -1); + const int finished = sqlite3_backup_finish(backup); + if (copied != SQLITE_DONE || finished != SQLITE_OK) + throw std::runtime_error("cannot copy telemetry database"); + Query runs(output.value, "SELECT count(*) FROM runs"); + Query events(output.value, "SELECT count(*) FROM events"); + if (sqlite3_step(runs.value) != SQLITE_ROW || sqlite3_step(events.value) != SQLITE_ROW) + throw std::runtime_error("cannot count copied telemetry"); + counts.telemetry_runs += + static_cast(sqlite3_column_int64(runs.value, 0)); + counts.telemetry_events += + static_cast(sqlite3_column_int64(events.value, 0)); + } catch (...) { + if (owned_empty_target) { + for (const auto* suffix : {"-wal", "-shm", "-journal", ""}) { + auto artifact = target; + artifact += suffix; + if (regular_state(artifact)) + fs::remove(artifact); + } + } + throw; + } + return; + } + Database output(target, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE); + verify_telemetry(output.value, false); + sql(output.value, "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE"); + try { + Query runs(input.value, "SELECT run_id FROM runs ORDER BY rowid"); + Query exists(output.value, "SELECT 1 FROM runs WHERE run_id=?1"); + for (;;) { + const int rc = sqlite3_step(runs.value); + if (rc == SQLITE_DONE) + break; + if (rc != SQLITE_ROW) + throw std::runtime_error(sqlite3_errmsg(input.value)); + const auto* id = reinterpret_cast(sqlite3_column_text(runs.value, 0)); + const auto length = sqlite3_column_bytes(runs.value, 0); + if (!id || !length) + throw std::runtime_error("invalid source telemetry run ID"); + const std::string_view run_id(id, static_cast(length)); + sqlite3_reset(exists.value); + sqlite3_clear_bindings(exists.value); + sqlite3_bind_text(exists.value, 1, id, length, SQLITE_TRANSIENT); + if (sqlite3_step(exists.value) == SQLITE_ROW) + continue; + counts.telemetry_runs += + copy_rows(input.value, output.value, "SELECT * FROM runs WHERE run_id=?1", + "INSERT INTO runs VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", run_id); + counts.telemetry_events += copy_rows( + input.value, output.value, + "SELECT run_id,type,name,severity,message,backend,worker,span_id," + "parent_span_id,time_ns,duration_ns,bytes,value,unit,truncated " + "FROM events WHERE run_id=?1 ORDER BY event_id", + "INSERT INTO events(run_id,type,name,severity,message,backend,worker,span_id," + "parent_span_id,time_ns,duration_ns,bytes,value,unit,truncated) " + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + run_id); + copy_rows(input.value, output.value, "SELECT * FROM metrics WHERE run_id=?1", + "INSERT INTO metrics VALUES(?,?,?,?)", run_id); + copy_rows(input.value, output.value, "SELECT * FROM parameters WHERE run_id=?1", + "INSERT INTO parameters VALUES(?,?,?,?)", run_id); + } + sql(output.value, "COMMIT"); + } catch (...) { + sqlite3_exec(output.value, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } +} + +/// 和 scan summary 一样保留结构与纯文本语义,颜色仅加强层级。 +/// Mirror scan summary's plain-text hierarchy; color only reinforces it. +void report(const Counts& counts, MergeOptions options, std::ostream& out) { + if (!options.summary) + return; + if (!options.pretty) { + out << "merge_workspaces=" << counts.workspaces + << " merge_model_contributions=" << counts.model_parts + << " merge_setup_contributions=" << counts.setup_parts + << " merge_cache_imported=" << counts.cache_imported + << " merge_telemetry_runs=" << counts.telemetry_runs + << " merge_telemetry_events=" << counts.telemetry_events + << " merge_ignore_rules_added=" << counts.ignore_rules_added + << " merge_sources_removed=" << counts.removed << '\n'; + return; + } + const auto cyan = options.color ? "\033[1;36m" : ""; + const auto green = options.color ? "\033[1;32m" : ""; + const auto reset = options.color ? "\033[0m" : ""; + out << '\n' + << cyan << "Merge summary" << reset << "\n" + << "========================================================================\n\n" + << cyan << "[ Sources ]" << reset << "\n" + << " Workspaces " << green << counts.workspaces << reset << " imported | " + << counts.removed << " removed\n\n" + << cyan << "[ State cache ]" << reset << "\n" + << " Absolute entries " << counts.cache_imported + << " imported (checked against file stamps only when reused)\n\n" + << cyan << "[ Online routing model ]" << reset << "\n" + << " Contributions " << counts.model_parts << " model | " << counts.setup_parts + << " setup considered (revision-deduplicated)\n\n" + << cyan << "[ Workspace settings ]" << reset << "\n" + << " Configuration preserved\n" + << " Ignore " << counts.ignore_rules_added << " missing rule(s) appended\n\n" + << cyan << "[ Telemetry ]" << reset << "\n" + << " Imported " << counts.telemetry_runs << " runs | " + << counts.telemetry_events << " events\n"; +} +} // namespace + +int merge_workspaces(const fs::path& destination, const fs::path& source, MergeOptions options, + std::ostream& diagnostics) { + const auto target = fs::canonical(destination); + const auto inputs = sources(target, source, options.recursive); + const auto state = target / ".same"; + if (fs::exists(fs::symlink_status(state))) { + if (!fs::is_directory(fs::symlink_status(state)) || is_reparse_point(state)) + throw std::runtime_error("destination .same must be a real directory"); + } else + fs::create_directory(state); + Counts counts; + std::vector> cleanup; + { + WorkspaceLock target_workspace(target); + RunLock target_lock(state / "run.lock"); + for (const auto& input : inputs) { + WorkspaceLock source_workspace(input); + RunLock source_lock(input / ".same" / "run.lock"); + merge_model(target, input, counts); + merge_state(target, input, counts); + merge_telemetry(target, input, counts); + merge_ignore(target, input, counts, diagnostics); + if (options.move) + cleanup.emplace_back(input, snapshot_workspace_state(input)); + ++counts.workspaces; + } + } + for (const auto& [input, expected] : cleanup) { + clean_workspace_if_unchanged(input, expected); + ++counts.removed; + } + report(counts, options, diagnostics); + return 0; +} +} // namespace same diff --git a/src/model_store.cpp b/src/model_store.cpp index 4e61cc9..819bad0 100644 --- a/src/model_store.cpp +++ b/src/model_store.cpp @@ -1,4 +1,6 @@ #include "same/model_store.hpp" +#include +#include #include #include #include @@ -16,7 +18,7 @@ namespace same { namespace { /// 格式与大小固定,不依赖本机字节序或结构填充。 / Fixed format independent of endianness/padding. -constexpr int schema_version = 2; +constexpr int schema_version = 4; constexpr std::size_t payload_size = 3 * (30 * 8 + 8); /// 拒绝链接和特殊文件,包括 SQLite 边车。 / Reject links and special files, including SQLite /// sidecars. @@ -70,6 +72,25 @@ bool valid_setup(const SetupHistory& h) { std::isfinite(h.max_ms) && h.last_ms > 0 && h.mean_ms > 0 && h.max_ms <= 1e9 && h.last_ms <= h.max_ms && h.mean_ms <= h.max_ms; } +/// 初始化历史沿用固定 32 字节载荷。 / Setup history retains its fixed 32-byte payload. +std::vector encode_setup(const SetupHistory& history) { + std::vector bytes; + bytes.reserve(32); + append(bytes, std::bit_cast(history.last_ms)); + append(bytes, std::bit_cast(history.mean_ms)); + append(bytes, std::bit_cast(history.max_ms)); + append(bytes, history.samples); + return bytes; +} +/// 解码来源账本及聚合表的同一表示。 / Decode the shared ledger/materialization format. +SetupHistory decode_setup(const unsigned char* bytes) { + SetupHistory h; + h.last_ms = std::bit_cast(take(bytes)); + h.mean_ms = std::bit_cast(take(bytes)); + h.max_ms = std::bit_cast(take(bytes)); + h.samples = take(bytes); + return h; +} /// 显式字段序列化,不保存 ABI。 / Explicit field serialization, never ABI serialization. std::vector encode(const detail::OnlineModel::State& state) { std::vector bytes; @@ -110,10 +131,10 @@ detail::OnlineModel::State decode(const unsigned char* bytes) { /// 存储种类决定表与版本列,避免调用方组合不一致的 SQL。 /// Record kind selects the table and version column, preventing mismatched SQL combinations. enum class RecordKind { model, setup }; -/// 两种学习载荷共享原子替换与有界裁剪;失败由调用方回滚并形成诊断。 -/// Both payloads share atomic replacement and bounded pruning; caller rolls back on failure. -void replace_payload(sqlite3* db, std::string_view key, const unsigned char* payload, - std::size_t payload_bytes, RecordKind kind) { +/// 聚合表写入处于调用方事务中;来源账本裁剪同步执行。 +/// Materialized writes run inside the caller transaction and prune orphan ledger rows. +void write_payload(sqlite3* db, std::string_view key, const unsigned char* payload, + std::size_t payload_bytes, RecordKind kind) { const bool model = kind == RecordKind::model; const char* insert_sql = model ? "INSERT OR REPLACE INTO models(key,version,payload) VALUES(?,?,?)" @@ -122,7 +143,6 @@ void replace_payload(sqlite3* db, std::string_view key, const unsigned char* pay "WHERE key<>? ORDER BY key LIMIT -1 OFFSET 31)" : "DELETE FROM setups WHERE key IN (SELECT key FROM setups " "WHERE key<>? ORDER BY key LIMIT -1 OFFSET 31)"; - execute(db, "BEGIN IMMEDIATE"); Statement insert; check(db, sqlite3_prepare_v2(db, insert_sql, -1, &insert.value, nullptr)); check(db, sqlite3_bind_blob(insert.value, 1, key.data(), static_cast(key.size()), @@ -140,13 +160,174 @@ void replace_payload(sqlite3* db, std::string_view key, const unsigned char* pay check(db, sqlite3_bind_blob(prune.value, 1, key.data(), static_cast(key.size()), SQLITE_TRANSIENT)); check(db, sqlite3_step(prune.value)); - execute(db, "COMMIT"); + execute(db, model ? "DELETE FROM model_parts WHERE key NOT IN (SELECT key FROM models)" + : "DELETE FROM setup_parts WHERE key NOT IN (SELECT key FROM setups)"); +} +/// 固定 BLOB 列复制为拥有值,供来源账本遍历。 / Copy a BLOB column into owned storage. +std::string blob(sqlite3_stmt* row, int column) { + const auto size = sqlite3_column_bytes(row, column); + const auto* data = static_cast(sqlite3_column_blob(row, column)); + return size ? std::string(data, static_cast(size)) : std::string{}; +} +/// 路径标识后接 NUL 与随机实例号;同路径重建工作区不会复用旧来源。 +/// Path plus NUL and a random instance token prevents identity reuse after --move. +std::string new_source_id(std::string_view root) { + std::array random{}; + sqlite3_randomness(static_cast(random.size()), random.data()); + std::string result(root); + result.push_back('\0'); + result.append(reinterpret_cast(random.data()), random.size()); + if (result.size() > 4096) + throw std::runtime_error("model source identity exceeds 4096 bytes"); + return result; +} +/// 逐行验证来源状态,避免合并损坏的模型。 / Validate each contribution before merging. +std::vector model_parts(sqlite3* db, std::string_view key = {}) { + const char* sql = key.empty() + ? "SELECT source,key,revision,attenuation,payload FROM model_parts " + "ORDER BY source,key" + : "SELECT source,key,revision,attenuation,payload FROM model_parts " + "WHERE key=?1 ORDER BY source"; + Statement query; + check(db, sqlite3_prepare_v2(db, sql, -1, &query.value, nullptr)); + if (!key.empty()) + check(db, sqlite3_bind_blob(query.value, 1, key.data(), static_cast(key.size()), + SQLITE_TRANSIENT)); + std::vector result; + for (;;) { + const int rc = sqlite3_step(query.value); + if (rc == SQLITE_DONE) + break; + if (rc != SQLITE_ROW) + check(db, rc); + ModelContribution part; + part.source = blob(query.value, 0); + part.key = blob(query.value, 1); + const auto revision = sqlite3_column_int64(query.value, 2); + part.attenuation = sqlite3_column_double(query.value, 3); + if (part.source.empty() || part.key.empty() || part.key.size() > 4096 || revision <= 0 || + !std::isfinite(part.attenuation) || part.attenuation <= 0 || part.attenuation > 1 || + sqlite3_column_bytes(query.value, 4) != payload_size) + throw std::runtime_error("invalid model contribution"); + part.revision = static_cast(revision); + part.state = decode(static_cast(sqlite3_column_blob(query.value, 4))); + if (!detail::OnlineModel::valid_state(part.state)) + throw std::runtime_error("invalid model contribution statistics"); + result.push_back(std::move(part)); + } + return result; +} +/// 同一主键上的 replace 不增加来源数;外部 revision 必须已验证。 +/// Replace a contribution at its stable source/key identity. +void put_model_part(sqlite3* db, const ModelContribution& part) { + const auto bytes = encode(part.state); + Statement insert; + check(db, sqlite3_prepare_v2(db, "INSERT OR REPLACE INTO model_parts VALUES(?1,?2,?3,?4,?5)", + -1, &insert.value, nullptr)); + check(db, sqlite3_bind_blob(insert.value, 1, part.source.data(), + static_cast(part.source.size()), SQLITE_TRANSIENT)); + check(db, sqlite3_bind_blob(insert.value, 2, part.key.data(), static_cast(part.key.size()), + SQLITE_TRANSIENT)); + check(db, sqlite3_bind_int64(insert.value, 3, static_cast(part.revision))); + check(db, sqlite3_bind_double(insert.value, 4, part.attenuation)); + check(db, sqlite3_bind_blob(insert.value, 5, bytes.data(), static_cast(bytes.size()), + SQLITE_TRANSIENT)); + check(db, sqlite3_step(insert.value)); +} +/// 聚合只从来源账本重建,杜绝旧来源残留和重复相加。 +/// Rematerialize solely from provenance so replaced sources cannot leave stale evidence. +detail::OnlineModel::State materialize_model(sqlite3* db, std::string_view key) { + detail::OnlineModel::State aggregate{}; + for (const auto& part : model_parts(db, key)) { + detail::OnlineModel effective; + if (!effective.initialize(part.state, part.attenuation) || + !detail::OnlineModel::merge(aggregate, effective.prior())) + throw std::runtime_error("model contribution aggregate rejected"); + } + const auto bytes = encode(aggregate); + write_payload(db, key, bytes.data(), bytes.size(), RecordKind::model); + return aggregate; +} +/// 初始化历史来源使用相同的 revision 协议。 / Setup histories use the same revision protocol. +std::vector setup_parts(sqlite3* db, std::string_view key = {}) { + const char* sql = key.empty() ? "SELECT source,key,revision,payload FROM setup_parts " + "ORDER BY source,key" + : "SELECT source,key,revision,payload FROM setup_parts " + "WHERE key=?1 ORDER BY source"; + Statement query; + check(db, sqlite3_prepare_v2(db, sql, -1, &query.value, nullptr)); + if (!key.empty()) + check(db, sqlite3_bind_blob(query.value, 1, key.data(), static_cast(key.size()), + SQLITE_TRANSIENT)); + std::vector result; + for (;;) { + const int rc = sqlite3_step(query.value); + if (rc == SQLITE_DONE) + break; + if (rc != SQLITE_ROW) + check(db, rc); + SetupContribution part; + part.source = blob(query.value, 0); + part.key = blob(query.value, 1); + const auto revision = sqlite3_column_int64(query.value, 2); + if (part.source.empty() || part.key.empty() || part.key.size() > 4096 || revision <= 0 || + sqlite3_column_bytes(query.value, 3) != 32) + throw std::runtime_error("invalid setup contribution"); + part.revision = static_cast(revision); + part.history = + decode_setup(static_cast(sqlite3_column_blob(query.value, 3))); + if (!valid_setup(part.history)) + throw std::runtime_error("invalid setup contribution history"); + result.push_back(std::move(part)); + } + return result; +} +/// 保存完整来源初始化历史。 / Save one complete source setup history. +void put_setup_part(sqlite3* db, const SetupContribution& part) { + const auto bytes = encode_setup(part.history); + Statement insert; + check(db, sqlite3_prepare_v2(db, "INSERT OR REPLACE INTO setup_parts VALUES(?1,?2,?3,?4)", -1, + &insert.value, nullptr)); + check(db, sqlite3_bind_blob(insert.value, 1, part.source.data(), + static_cast(part.source.size()), SQLITE_TRANSIENT)); + check(db, sqlite3_bind_blob(insert.value, 2, part.key.data(), static_cast(part.key.size()), + SQLITE_TRANSIENT)); + check(db, sqlite3_bind_int64(insert.value, 3, static_cast(part.revision))); + check(db, sqlite3_bind_blob(insert.value, 4, bytes.data(), static_cast(bytes.size()), + SQLITE_TRANSIENT)); + check(db, sqlite3_step(insert.value)); +} +/// 样本数加权均值与保守最大值;last 取稳定来源顺序中的末项。 +/// Weighted mean and conservative maximum; last follows stable source order. +SetupHistory materialize_setup(sqlite3* db, std::string_view key) { + SetupHistory aggregate{}; + for (const auto& part : setup_parts(db, key)) { + const auto& h = part.history; + if (h.samples > UINT64_MAX - aggregate.samples) + throw std::runtime_error("setup contribution count overflow"); + const auto total = aggregate.samples + h.samples; + aggregate.mean_ms = (aggregate.mean_ms * static_cast(aggregate.samples) + + h.mean_ms * static_cast(h.samples)) / + static_cast(total); + aggregate.samples = total; + aggregate.last_ms = h.last_ms; + aggregate.max_ms = std::max(aggregate.max_ms, h.max_ms); + } + if (!valid_setup(aggregate)) + throw std::runtime_error("setup contribution aggregate rejected"); + const auto bytes = encode_setup(aggregate); + write_payload(db, key, bytes.data(), bytes.size(), RecordKind::setup); + return aggregate; } } // namespace /// 冷路径连接;关闭自动回滚未提交事务。 / Cold-path connection; closing rolls back transactions. struct ModelStore::Impl { sqlite3* db{}; std::string diagnostic; + /// 仅用于迁移旧路径身份与检测复制工作区。 / Root identity for legacy migration/copy detection. + std::string root_source; + /// 当前存储根的稳定绝对身份。 / Stable absolute identity of this store's workspace. + std::string local_source; /// 启动快照支持并发只读查询。 / Startup snapshot supports concurrent read-only lookup. std::map> entries; /// 初始化历史冷路径快照。 / Cold-path setup history snapshot. @@ -157,6 +338,9 @@ struct ModelStore::Impl { }; ModelStore::ModelStore(const std::filesystem::path& path) : impl_(std::make_unique()) { try { + const auto source_path = std::filesystem::canonical(path.parent_path()).generic_u8string(); + impl_->root_source.assign(reinterpret_cast(source_path.data()), + source_path.size()); guard_path(path); for (const auto* suffix : {"-journal", "-wal", "-shm"}) { auto sidecar = path; @@ -189,7 +373,7 @@ ModelStore::ModelStore(const std::filesystem::path& path) : impl_(std::make_uniq if (sqlite3_step(version.value) != SQLITE_ROW) throw std::runtime_error("missing model schema version"); const int v = sqlite3_column_int(version.value, 0); - if (v != 0 && v != 1 && v != schema_version) + if (v < 0 || v > schema_version) throw std::runtime_error("unsupported model schema version"); sqlite3_finalize(version.value); version.value = nullptr; @@ -202,7 +386,7 @@ ModelStore::ModelStore(const std::filesystem::path& path) : impl_(std::make_uniq } else if (application != 1396788556) { throw std::runtime_error("unowned model database"); } - execute(impl_->db, "PRAGMA max_page_count=256"); + execute(impl_->db, "PRAGMA max_page_count=16384"); if (v == 0) execute(impl_->db, "BEGIN IMMEDIATE; CREATE TABLE models(key BLOB PRIMARY KEY NOT " "NULL, version INTEGER NOT NULL, " @@ -213,6 +397,117 @@ ModelStore::ModelStore(const std::filesystem::path& path) : impl_(std::make_uniq "BEGIN IMMEDIATE; CREATE TABLE setups(key BLOB PRIMARY KEY NOT NULL, " "payload BLOB NOT NULL CHECK(length(payload)=32)) WITHOUT ROWID; PRAGMA " "user_version=2; COMMIT;"); + if (v < 3) { + execute(impl_->db, + "BEGIN IMMEDIATE;" + "CREATE TABLE model_parts(source BLOB NOT NULL,key BLOB NOT NULL," + "revision INTEGER NOT NULL CHECK(revision>0),attenuation REAL NOT NULL " + "CHECK(attenuation>0 AND attenuation<=1),payload BLOB NOT NULL " + "CHECK(length(payload)=744),PRIMARY KEY(source,key)) WITHOUT ROWID;" + "CREATE TABLE setup_parts(source BLOB NOT NULL,key BLOB NOT NULL," + "revision INTEGER NOT NULL CHECK(revision>0),payload BLOB NOT NULL " + "CHECK(length(payload)=32),PRIMARY KEY(source,key)) WITHOUT ROWID;"); + try { + Statement models; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, + "INSERT INTO model_parts SELECT ?1,key,1,1.0,payload " + "FROM models", + -1, &models.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(models.value, 1, impl_->root_source.data(), + static_cast(impl_->root_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(models.value)); + Statement setups; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, + "INSERT INTO setup_parts SELECT ?1,key,1,payload " + "FROM setups", + -1, &setups.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(setups.value, 1, impl_->root_source.data(), + static_cast(impl_->root_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(setups.value)); + execute(impl_->db, "PRAGMA user_version=3; COMMIT;"); + } catch (...) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } + } + if (v < 4) { + const auto identity = new_source_id(impl_->root_source); + execute(impl_->db, + "BEGIN IMMEDIATE; CREATE TABLE model_meta(id INTEGER PRIMARY KEY CHECK(id=1)," + "root BLOB NOT NULL,local_source BLOB NOT NULL);"); + try { + Statement meta; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, "INSERT INTO model_meta VALUES(1,?1,?2)", -1, + &meta.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(meta.value, 1, impl_->root_source.data(), + static_cast(impl_->root_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, + sqlite3_bind_blob(meta.value, 2, identity.data(), + static_cast(identity.size()), SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(meta.value)); + for (const char* table : {"model_parts", "setup_parts"}) { + const auto sql = + std::string("UPDATE ") + table + " SET source=?1 WHERE source=?2"; + Statement update; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, sql.c_str(), -1, &update.value, nullptr)); + check(impl_->db, + sqlite3_bind_blob(update.value, 1, identity.data(), + static_cast(identity.size()), SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_bind_blob(update.value, 2, impl_->root_source.data(), + static_cast(impl_->root_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(update.value)); + } + execute(impl_->db, "PRAGMA user_version=4; COMMIT;"); + impl_->local_source = identity; + } catch (...) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } + } else { + Statement meta; + check(impl_->db, sqlite3_prepare_v2(impl_->db, + "SELECT root,local_source FROM model_meta WHERE " + "id=1", + -1, &meta.value, nullptr)); + if (sqlite3_step(meta.value) != SQLITE_ROW) + throw std::runtime_error("missing model workspace identity"); + const auto owner_root = blob(meta.value, 0); + impl_->local_source = blob(meta.value, 1); + sqlite3_finalize(meta.value); + meta.value = nullptr; + if (impl_->local_source.empty()) + throw std::runtime_error("invalid model workspace identity"); + if (owner_root != impl_->root_source) { + impl_->local_source = new_source_id(impl_->root_source); + execute(impl_->db, "BEGIN IMMEDIATE"); + try { + Statement update; + check(impl_->db, sqlite3_prepare_v2(impl_->db, + "UPDATE model_meta SET root=?1," + "local_source=?2 WHERE id=1", + -1, &update.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(update.value, 1, impl_->root_source.data(), + static_cast(impl_->root_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_bind_blob(update.value, 2, impl_->local_source.data(), + static_cast(impl_->local_source.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(update.value)); + execute(impl_->db, "COMMIT"); + } catch (...) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } + } + } // 仅在确认归属与版本后设置日志;学习状态允许断电丢失最近提交。 // Change journaling only after ownership/version checks; recent learning may be lost on // power failure. @@ -304,8 +599,114 @@ bool ModelStore::save(std::string_view key, const detail::OnlineModel::State& st throw std::runtime_error("invalid model key length"); if (!detail::OnlineModel::valid_state(state)) throw std::runtime_error("invalid model statistics"); - const auto bytes = encode(state); - replace_payload(impl_->db, key, bytes.data(), bytes.size(), RecordKind::model); + execute(impl_->db, "BEGIN IMMEDIATE"); + std::uint64_t revision = 1; + for (const auto& part : model_parts(impl_->db, key)) + if (part.source == impl_->local_source) { + revision = part.revision + 1; + break; + } + if (revision > static_cast(INT64_MAX)) + throw std::runtime_error("model source revision overflow"); + put_model_part(impl_->db, {impl_->local_source, std::string(key), revision, 1, state}); + const auto aggregate = materialize_model(impl_->db, key); + execute(impl_->db, "COMMIT"); + impl_->entries[std::string(key)] = aggregate; + impl_->diagnostic.clear(); + return true; + } catch (const std::exception& e) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + impl_->diagnostic = e.what(); + return false; + } +} +bool ModelStore::save_delta(std::string_view key, const detail::OnlineModel::State& delta, + double decay) { + if (!impl_->db) + return false; + try { + if (key.empty() || key.size() > 4096 || !detail::OnlineModel::valid_state(delta) || + !std::isfinite(decay) || decay <= 0 || decay > 1) + throw std::runtime_error("invalid model delta/key/decay"); + execute(impl_->db, "BEGIN IMMEDIATE"); + bool local = false; + for (auto part : model_parts(impl_->db, key)) { + const auto next_attenuation = part.attenuation * decay; + detail::OnlineModel effective; + if (!effective.initialize(part.state, next_attenuation)) + throw std::runtime_error("model source decay rejected"); + if (part.source == impl_->local_source) { + part.state = effective.prior(); + if (part.revision == static_cast(INT64_MAX) || + !detail::OnlineModel::merge(part.state, delta)) + throw std::runtime_error("local model delta rejected"); + ++part.revision; + part.attenuation = 1; + local = true; + } else + part.attenuation = next_attenuation; + put_model_part(impl_->db, part); + } + if (!local) + put_model_part(impl_->db, {impl_->local_source, std::string(key), 1, 1, delta}); + const auto aggregate = materialize_model(impl_->db, key); + execute(impl_->db, "COMMIT"); + impl_->entries[std::string(key)] = aggregate; + impl_->diagnostic.clear(); + return true; + } catch (const std::exception& e) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + impl_->diagnostic = e.what(); + return false; + } +} +std::vector ModelStore::contributions() const { + if (!impl_->db) + throw std::runtime_error("model store unavailable"); + return model_parts(impl_->db); +} +bool ModelStore::merge_contribution(const ModelContribution& contribution) { + if (!impl_->db) + return false; + try { + if (contribution.source.empty() || contribution.source.size() > 4096 || + contribution.key.empty() || contribution.key.size() > 4096 || !contribution.revision || + contribution.revision > static_cast(INT64_MAX) || + !std::isfinite(contribution.attenuation) || contribution.attenuation <= 0 || + contribution.attenuation > 1 || !detail::OnlineModel::valid_state(contribution.state)) + throw std::runtime_error("invalid imported model contribution"); + execute(impl_->db, "BEGIN IMMEDIATE"); + bool newer = true; + if (contribution.source == impl_->local_source) + newer = false; + for (const auto& existing : model_parts(impl_->db, contribution.key)) + if (existing.source == contribution.source) { + newer = contribution.revision > existing.revision; + break; + } + std::optional aggregate; + if (newer) { + const auto separator = contribution.source.find('\0'); + if (separator != std::string::npos && + contribution.source.substr(0, separator) != impl_->root_source) { + Statement legacy; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, + "DELETE FROM model_parts WHERE source=?1 AND key=?2", -1, + &legacy.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(legacy.value, 1, contribution.source.data(), + static_cast(separator), SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_bind_blob(legacy.value, 2, contribution.key.data(), + static_cast(contribution.key.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(legacy.value)); + } + put_model_part(impl_->db, contribution); + aggregate = materialize_model(impl_->db, contribution.key); + } + execute(impl_->db, "COMMIT"); + if (aggregate) + impl_->entries[contribution.key] = *aggregate; impl_->diagnostic.clear(); return true; } catch (const std::exception& e) { @@ -326,13 +727,108 @@ bool ModelStore::save_setup(std::string_view key, const SetupHistory& history) { try { if (key.empty() || key.size() > 4096 || !valid_setup(history)) throw std::runtime_error("invalid setup history/key"); - std::vector bytes; - bytes.reserve(32); - append(bytes, std::bit_cast(history.last_ms)); - append(bytes, std::bit_cast(history.mean_ms)); - append(bytes, std::bit_cast(history.max_ms)); - append(bytes, history.samples); - replace_payload(impl_->db, key, bytes.data(), bytes.size(), RecordKind::setup); + execute(impl_->db, "BEGIN IMMEDIATE"); + std::uint64_t revision = 1; + for (const auto& part : setup_parts(impl_->db, key)) + if (part.source == impl_->local_source) { + revision = part.revision + 1; + break; + } + if (revision > static_cast(INT64_MAX)) + throw std::runtime_error("setup source revision overflow"); + put_setup_part(impl_->db, {impl_->local_source, std::string(key), revision, history}); + const auto aggregate = materialize_setup(impl_->db, key); + execute(impl_->db, "COMMIT"); + impl_->setups[std::string(key)] = aggregate; + impl_->diagnostic.clear(); + return true; + } catch (const std::exception& e) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + impl_->diagnostic = e.what(); + return false; + } +} +bool ModelStore::save_setup_delta(std::string_view key, const SetupHistory& delta) { + if (!impl_->db) + return false; + try { + if (key.empty() || key.size() > 4096 || !valid_setup(delta)) + throw std::runtime_error("invalid setup delta/key"); + execute(impl_->db, "BEGIN IMMEDIATE"); + auto local = SetupContribution{impl_->local_source, std::string(key), 1, delta}; + for (const auto& part : setup_parts(impl_->db, key)) + if (part.source == impl_->local_source) { + if (part.revision == static_cast(INT64_MAX) || + delta.samples > UINT64_MAX - part.history.samples) + throw std::runtime_error("setup source overflow"); + const auto total = part.history.samples + delta.samples; + local.revision = part.revision + 1; + local.history.mean_ms = + (part.history.mean_ms * static_cast(part.history.samples) + + delta.mean_ms * static_cast(delta.samples)) / + static_cast(total); + local.history.samples = total; + local.history.max_ms = std::max(part.history.max_ms, delta.max_ms); + break; + } + put_setup_part(impl_->db, local); + const auto aggregate = materialize_setup(impl_->db, key); + execute(impl_->db, "COMMIT"); + impl_->setups[std::string(key)] = aggregate; + impl_->diagnostic.clear(); + return true; + } catch (const std::exception& e) { + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + impl_->diagnostic = e.what(); + return false; + } +} +std::vector ModelStore::setup_contributions() const { + if (!impl_->db) + throw std::runtime_error("model store unavailable"); + return setup_parts(impl_->db); +} +bool ModelStore::merge_setup_contribution(const SetupContribution& contribution) { + if (!impl_->db) + return false; + try { + if (contribution.source.empty() || contribution.source.size() > 4096 || + contribution.key.empty() || contribution.key.size() > 4096 || !contribution.revision || + contribution.revision > static_cast(INT64_MAX) || + !valid_setup(contribution.history)) + throw std::runtime_error("invalid imported setup contribution"); + execute(impl_->db, "BEGIN IMMEDIATE"); + bool newer = true; + if (contribution.source == impl_->local_source) + newer = false; + for (const auto& existing : setup_parts(impl_->db, contribution.key)) + if (existing.source == contribution.source) { + newer = contribution.revision > existing.revision; + break; + } + std::optional aggregate; + if (newer) { + const auto separator = contribution.source.find('\0'); + if (separator != std::string::npos && + contribution.source.substr(0, separator) != impl_->root_source) { + Statement legacy; + check(impl_->db, + sqlite3_prepare_v2(impl_->db, + "DELETE FROM setup_parts WHERE source=?1 AND key=?2", -1, + &legacy.value, nullptr)); + check(impl_->db, sqlite3_bind_blob(legacy.value, 1, contribution.source.data(), + static_cast(separator), SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_bind_blob(legacy.value, 2, contribution.key.data(), + static_cast(contribution.key.size()), + SQLITE_TRANSIENT)); + check(impl_->db, sqlite3_step(legacy.value)); + } + put_setup_part(impl_->db, contribution); + aggregate = materialize_setup(impl_->db, contribution.key); + } + execute(impl_->db, "COMMIT"); + if (aggregate) + impl_->setups[contribution.key] = *aggregate; impl_->diagnostic.clear(); return true; } catch (const std::exception& e) { diff --git a/src/resources.cpp b/src/resources.cpp index 2a7c770..3325ae8 100644 --- a/src/resources.cpp +++ b/src/resources.cpp @@ -179,7 +179,14 @@ detail::OnlineModel::Context Resources::context(const Worker& worker, BackendKin std::pair Resources::choose_backend(Worker& worker, std::uint64_t bytes, bool hash, - const std::array& candidates) { + const std::array& candidates, + std::optional training_backend) { + if (training_backend) { + const auto requested = *training_backend; + const auto slot = static_cast(requested); + return {slot < candidates.size() && candidates[slot] ? requested : BackendKind::cpu, + Reason::fixed}; + } if (hash && (!bytes || bytes < worker.gpu_floor)) return {BackendKind::cpu, Reason::size}; if (policy_ != BackendPolicy::automatic || !hash) { @@ -242,8 +249,8 @@ Resources::choose_backend(Worker& worker, std::uint64_t bytes, bool hash, return known ? std::pair{best, Reason::model} : std::pair{preferred, Reason::fixed}; } -bool Resources::begin_cold(const Worker& worker, double estimate) { - if (policy_ != BackendPolicy::automatic || !pgo_) +bool Resources::begin_cold(const Worker& worker, double estimate, bool training) { + if (training || policy_ != BackendPolicy::automatic || !pgo_) return true; bool idle = false; if (!cold_start_busy_.compare_exchange_strong(idle, true, std::memory_order_acquire)) @@ -266,7 +273,7 @@ void Resources::end_cold() { cold_start_busy_.store(false, std::memory_order_release); } -bool Resources::discover_igpu(Worker& worker) { +bool Resources::discover_igpu(Worker& worker, bool training) { bool idle = false; if (!igpu_busy_.compare_exchange_strong(idle, true, std::memory_order_acquire)) { ++worker.igpu_busy; @@ -274,7 +281,7 @@ bool Resources::discover_igpu(Worker& worker) { } try { if (!igpu_probed_) { - if (!begin_cold(worker, igpu_bootstrap_ms_)) { + if (!begin_cold(worker, igpu_bootstrap_ms_, training)) { ++igpu_deferred_; igpu_busy_.store(false, std::memory_order_release); return false; @@ -328,8 +335,8 @@ bool Resources::discover_igpu(Worker& worker) { } } -bool Resources::admit_cold_igpu(const Worker& worker) const { - if (policy_ != BackendPolicy::automatic || !pgo_ || igpu_setup_estimate_ms_ == 0) +bool Resources::admit_cold_igpu(const Worker& worker, bool training) const { + if (training || policy_ != BackendPolicy::automatic || !pgo_ || igpu_setup_estimate_ms_ == 0) return true; const auto cpu = worker.model.predict(BackendKind::cpu, worker.candidate_contexts[0]); const auto igpu = worker.model.predict(BackendKind::igpu, worker.candidate_contexts[2]); @@ -343,7 +350,7 @@ bool Resources::admit_cold_igpu(const Worker& worker) const { igpu_setup_estimate_ms_; } -bool Resources::select_igpu(Worker& worker) { +bool Resources::select_igpu(Worker& worker, bool training) { bool idle = false; if (!igpu_busy_.compare_exchange_strong(idle, true, std::memory_order_acquire)) { ++worker.igpu_busy; @@ -363,12 +370,12 @@ bool Resources::select_igpu(Worker& worker) { }; try { if (!igpu_attempted_) { - if (!begin_cold(worker, igpu_setup_estimate_ms_)) { + if (!begin_cold(worker, igpu_setup_estimate_ms_, training)) { ++igpu_deferred_; igpu_busy_.store(false, std::memory_order_release); return false; } - if (!admit_cold_igpu(worker)) { + if (!admit_cold_igpu(worker, training)) { end_cold(); ++igpu_deferred_; igpu_busy_.store(false, std::memory_order_release); @@ -485,7 +492,7 @@ bool Resources::prepare_gpu(Worker& worker) { } } -bool Resources::activate_cuda(Worker& worker) { +bool Resources::activate_cuda(Worker& worker, bool training) { bool owns_bootstrap = false; if (policy_ == BackendPolicy::automatic && !worker.gpu_attempted) { auto state = Bootstrap::cold; @@ -497,7 +504,7 @@ bool Resources::activate_cuda(Worker& worker) { } } const bool starting = !worker.gpu_attempted; - if (starting && !begin_cold(worker, cuda_bootstrap_ms_)) { + if (starting && !begin_cold(worker, cuda_bootstrap_ms_, training)) { ++cuda_deferred_; if (owns_bootstrap) bootstrap_.store(Bootstrap::cold, std::memory_order_release); @@ -534,6 +541,11 @@ bool Resources::activate_cuda(Worker& worker) { } void Resources::select_backend(Worker& worker, std::uint64_t bytes, bool hash) noexcept { + select_backend(worker, bytes, hash, std::nullopt); +} + +void Resources::select_backend(Worker& worker, std::uint64_t bytes, bool hash, + std::optional training_backend) noexcept { worker.startup_error = {}; worker.selected_backend = worker.compute.get(); try { @@ -556,11 +568,12 @@ void Resources::select_backend(Worker& worker, std::uint64_t bytes, bool hash) n for (unsigned slot = 0; slot < 3; ++slot) worker.candidate_contexts[slot] = context(worker, static_cast(slot), bytes); - const auto [kind, reason] = choose_backend(worker, bytes, hash, candidates); + const auto [kind, reason] = + choose_backend(worker, bytes, hash, candidates, training_backend); const auto slot = static_cast(kind); const bool known_device = worker.prior_loaded[slot]; if (kind == BackendKind::igpu && igpu_probe_ && !known_device) { - if (!discover_igpu(worker)) { + if (!discover_igpu(worker, training_backend.has_value())) { candidates[slot] = false; ++worker.excluded[slot]; } @@ -568,7 +581,8 @@ void Resources::select_backend(Worker& worker, std::uint64_t bytes, bool hash) n } const bool activated = kind == BackendKind::cpu || - (kind == BackendKind::cuda ? activate_cuda(worker) : select_igpu(worker)); + (kind == BackendKind::cuda ? activate_cuda(worker, training_backend.has_value()) + : select_igpu(worker, training_backend.has_value())); if (!activated) { candidates[slot] = false; ++worker.excluded[slot]; diff --git a/src/store.cpp b/src/store.cpp index bbabd62..6ae0582 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -163,16 +163,20 @@ struct Store::Impl { /// 是否拥有尚未完成、需要回滚保护的扫描事务。 / Whether an unfinished scan transaction needs /// rollback protection. bool scanning{}; + /// 与扫描互斥的归档导入事务。 / Archive import transaction excludes scans. + bool importing{}; /// 本轮已见标记,仅在扫描事务中写入。 / This scan's seen marker, written within its /// transaction. sqlite3_int64 generation{}; /// 固定热路径语句,数量不随文件数增长。 / Fixed hot-path statements, independent of file count. std::unique_ptr lookup; + std::unique_ptr absolute_lookup; /// 缓存命中仅修改非索引列。 / Cache hits update only the non-indexed generation. std::unique_ptr seen; /// 一次主键查找完成文件戳验证和已见写入。 / Validate and mark seen with one primary-key lookup. std::unique_ptr unchanged; std::unique_ptr upsert; + std::unique_ptr absolute_upsert; std::unique_ptr representative; std::unique_ptr match; /// Store 构造失败时同样关闭已打开的连接。 / Close the connection even if Store construction @@ -181,7 +185,9 @@ struct Store::Impl { match.reset(); representative.reset(); upsert.reset(); + absolute_upsert.reset(); lookup.reset(); + absolute_lookup.reset(); seen.reset(); unchanged.reset(); if (db) @@ -204,8 +210,31 @@ Store::Store(const std::filesystem::path& path, std::size_t cache_bytes) if (!version.step()) throw std::runtime_error("Missing SQLite schema version"); const auto schema = version.integer(0); - if (schema != 0 && schema != 1) + if (schema != 0 && schema != 1 && schema != 2) throw std::runtime_error("Unsupported state database schema version"); + Statement owner(db, "PRAGMA application_id"); + if (!owner.step()) + throw std::runtime_error("Missing state database owner"); + const auto application = owner.integer(0); + if ((schema == 0 && application != 0) || (schema == 1 && application != 0) || + (schema == 2 && application != 1396788563)) + throw std::runtime_error("Foreign state database"); + if (schema == 0) { + Statement existing(db, "SELECT count(*) FROM sqlite_schema " + "WHERE name NOT LIKE 'sqlite_%'"); + if (!existing.step() || existing.integer(0) != 0) + throw std::runtime_error("Nonempty unowned state database"); + } + if (schema == 1) { + Statement shape(db, "SELECT " + "(SELECT count(*) FROM pragma_table_info('files') WHERE name IN " + "('path','size','identity','modified','changed','digest'," + "'generation')) + " + "(SELECT count(*) FROM pragma_table_info('scan_state') " + "WHERE name IN ('id','generation'))"); + if (!shape.step() || shape.integer(0) != 9) + throw std::runtime_error("Unrecognized legacy state database"); + } } exec(db, "PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL; PRAGMA temp_store=FILE; PRAGMA " "mmap_size=0;"); @@ -223,7 +252,12 @@ Store::Store(const std::filesystem::path& path, std::size_t cache_bytes) "CREATE INDEX IF NOT EXISTS files_content ON files(size,digest,path);" "CREATE TABLE IF NOT EXISTS scan_state(id INTEGER PRIMARY KEY " "CHECK(id=1),generation INTEGER NOT NULL);" - "INSERT OR IGNORE INTO scan_state VALUES(1,0); PRAGMA user_version=1; COMMIT;"); + "INSERT OR IGNORE INTO scan_state VALUES(1,0);" + "CREATE TABLE IF NOT EXISTS absolute_cache(" + "path BLOB PRIMARY KEY NOT NULL,size BLOB NOT NULL CHECK(length(size)=8)," + "identity BLOB NOT NULL,modified BLOB NOT NULL,changed BLOB NOT NULL," + "digest BLOB NOT NULL CHECK(length(digest)=32)) WITHOUT ROWID;" + "PRAGMA user_version=2; PRAGMA application_id=1396788563; COMMIT;"); } catch (...) { sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr); throw; @@ -239,6 +273,9 @@ Store::Store(const std::filesystem::path& path, std::size_t cache_bytes) impl_->lookup = std::make_unique( db, "SELECT path,size,identity,modified,changed,digest FROM files WHERE path=?1", SQLITE_PREPARE_PERSISTENT); + impl_->absolute_lookup = std::make_unique( + db, "SELECT path,size,identity,modified,changed,digest FROM absolute_cache WHERE path=?1", + SQLITE_PREPARE_PERSISTENT); impl_->seen = std::make_unique(db, "UPDATE files SET generation=?2 WHERE path=?1", SQLITE_PREPARE_PERSISTENT); impl_->unchanged = std::make_unique( @@ -253,6 +290,13 @@ Store::Store(const std::filesystem::path& path, std::size_t cache_bytes) "size=excluded.size,identity=excluded.identity,modified=excluded.modified," "changed=excluded.changed,digest=excluded.digest,generation=excluded.generation", SQLITE_PREPARE_PERSISTENT); + impl_->absolute_upsert = std::make_unique( + db, + "INSERT INTO absolute_cache(path,size,identity,modified,changed,digest)" + " VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(path) DO UPDATE SET " + "size=excluded.size,identity=excluded.identity,modified=excluded.modified," + "changed=excluded.changed,digest=excluded.digest", + SQLITE_PREPARE_PERSISTENT); impl_->representative = std::make_unique( db, "INSERT INTO representatives VALUES(?1,?2,?3,?4,?5,?6)", SQLITE_PREPARE_PERSISTENT); impl_->match = std::make_unique(db, "INSERT OR IGNORE INTO matches VALUES(?1,?2)", @@ -260,10 +304,11 @@ Store::Store(const std::filesystem::path& path, std::size_t cache_bytes) } Store::~Store() { rollback_scan(); + rollback_import(); } void Store::begin_scan() { - if (impl_->scanning) - throw std::logic_error("Scan already active"); + if (impl_->scanning || impl_->importing) + throw std::logic_error("State transaction already active"); exec(impl_->db, "BEGIN IMMEDIATE"); impl_->scanning = true; try { @@ -347,6 +392,55 @@ void Store::rollback_scan() noexcept { sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); impl_->scanning = false; } +std::optional Store::cached_absolute(std::string_view path) { + auto& query = *impl_->absolute_lookup; + StatementUse use(query); + query.string(1, path); + std::optional result; + if (query.step()) + result = query.record(); + use.finish(); + return result; +} +void Store::save_absolute(const FileRecord& record) { + if (!impl_->scanning && !impl_->importing) + throw std::logic_error("No active state transaction"); + auto& insert = *impl_->absolute_upsert; + StatementUse use(insert); + insert.bind_record(record); + insert.step(); + use.finish(); +} +void Store::begin_import() { + if (impl_->scanning || impl_->importing) + throw std::logic_error("State transaction already active"); + exec(impl_->db, "BEGIN IMMEDIATE"); + impl_->importing = true; +} +void Store::end_import() { + if (!impl_->importing) + throw std::logic_error("No active state import"); + exec(impl_->db, "COMMIT"); + impl_->importing = false; +} +void Store::rollback_import() noexcept { + if (!impl_->importing) + return; + sqlite3_exec(impl_->db, "ROLLBACK", nullptr, nullptr, nullptr); + impl_->importing = false; +} +void Store::visit_files(const std::function& visitor) { + Statement query(impl_->db, + "SELECT path,size,identity,modified,changed,digest FROM files ORDER BY path"); + while (query.step()) + visitor(query.record()); +} +void Store::visit_absolute(const std::function& visitor) { + Statement query(impl_->db, "SELECT path,size,identity,modified,changed,digest " + "FROM absolute_cache ORDER BY path"); + while (query.step()) + visitor(query.record()); +} void Store::visit_candidates(const std::function& visitor) { if (impl_->scanning) throw std::logic_error("Commit scan before comparing"); diff --git a/src/workspace.cpp b/src/workspace.cpp index 488be33..ca35722 100644 --- a/src/workspace.cpp +++ b/src/workspace.cpp @@ -1,6 +1,8 @@ #include "same/workspace.hpp" #include "same/config.hpp" +#include "same/files.hpp" #include "same/run_lock.hpp" +#include #include #include #include @@ -222,9 +224,54 @@ void remove_child(Node& parent, const fs::path& name) { throw std::system_error(errno, std::generic_category(), "remove cleanup child"); #endif } +/// 先后两次扫描文件版本,内容摘要防止粗粒度时间戳漏检同尺寸更改。 +/// Hash content and check stamps twice so coarse timestamps cannot hide same-size changes. +WorkspaceStateFile snapshot_file(const fs::path& path) { + FileReader reader(path); + const auto stamp = reader.stamp(); + auto hasher = make_cpu_compute()->hasher(); + std::vector block(64 * 1024); + std::uint64_t bytes = 0; + for (;;) { + const auto count = reader.read(block); + if (!count) + break; + if (count > stamp.size - bytes) + throw std::runtime_error("state file grew during --move verification"); + hasher->update(std::span(block).first(count)); + bytes += count; + } + if (bytes != stamp.size || reader.stamp() != stamp || stamp_path(path) != stamp) + throw std::runtime_error("state file changed during --move verification"); + const auto utf8 = path.filename().generic_u8string(); + return {std::string(reinterpret_cast(utf8.data()), utf8.size()), stamp, + hasher->finish()}; +} +/// 只采集普通直系文件;不认识的目录或链接使 --move 安全失败。 +/// Snapshot only ordinary direct files; unknown directories/links make --move fail safely. +WorkspaceStateSnapshot state_snapshot(const fs::path& state) { + WorkspaceStateSnapshot result; + for (const auto& entry : fs::directory_iterator(state)) { + const auto name = entry.path().filename(); + if (name == "run.lock") + continue; + if (!entry.is_regular_file() || linked(entry.path())) + throw std::runtime_error("cannot move workspace with non-regular state entries"); + result.push_back(snapshot_file(entry.path())); + } + std::sort(result.begin(), result.end(), + [](const auto& a, const auto& b) { return a.name < b.name; }); + return result; +} +/// 在已获取运行锁后比较所有状态项。 / Compare all state entries after acquiring the run lock. +void check_snapshot(const fs::path& state, const WorkspaceStateSnapshot& expected) { + if (state_snapshot(state) != expected) + throw std::runtime_error("source workspace changed after merge; refusing --move"); +} /// 在同一能力下发现和清除状态;递归时祖先对象不离开作用域。 /// Discover and clean under the same capability; DFS retains every ancestor in scope. -std::size_t clean_directory(Node& directory, bool recursive) { +std::size_t clean_directory(Node& directory, bool recursive, + const WorkspaceStateSnapshot* expected = nullptr) { std::size_t count = 0; for (const auto& name : directory.names()) { if (name != ".same") @@ -235,6 +282,8 @@ std::size_t clean_directory(Node& directory, bool recursive) { throw std::runtime_error(".same must be a non-link directory"); { RunLock lock(state.path / "run.lock"); + if (expected) + check_snapshot(state.path, *expected); for (const auto& child : state.names()) if (child != "run.lock") remove_child(state, child); @@ -250,6 +299,8 @@ std::size_t clean_directory(Node& directory, bool recursive) { throw std::system_error(errno, std::generic_category(), "workspace lifecycle lock"); Node state(directory.fd, name, directory.path / name); RunLock lock(state.fd, "run.lock"); + if (expected) + check_snapshot(state.path, *expected); for (const auto& child : state.names()) remove_child(state, child); if (unlinkat(directory.fd, ".same", AT_REMOVEDIR) != 0) @@ -285,19 +336,8 @@ std::size_t clean_directory(Node& directory, bool recursive) { } return count; } -} // namespace -void initialize_workspace(const fs::path& root) { - const auto resolved = checked_root(root); - WorkspaceLock lifecycle(resolved); - const auto state = resolved / ".same"; - if (!state_exists(state)) - fs::create_directory(state); - RunLock lock(state / "run.lock"); - write_missing(state / "config.toml", default_config()); - write_missing(state / "ignore", default_ignore); -} -std::size_t clean_workspace(const fs::path& root, bool recursive) { - const auto absolute = checked_root(root); +/// 破坏性操作使用同一套根到叶目录能力。 / Share root-to-leaf directory capabilities. +std::vector> cleanup_ancestors(const fs::path& absolute) { std::vector> ancestors; #ifdef _WIN32 ancestors.push_back(std::make_unique(absolute.root_path(), false)); @@ -314,6 +354,35 @@ std::size_t clean_workspace(const fs::path& root, bool recursive) { ancestors.push_back( std::make_unique(ancestors.back()->fd, part, ancestors.back()->path / part)); #endif + return ancestors; +} +} // namespace +void initialize_workspace(const fs::path& root) { + const auto resolved = checked_root(root); + WorkspaceLock lifecycle(resolved); + const auto state = resolved / ".same"; + if (!state_exists(state)) + fs::create_directory(state); + RunLock lock(state / "run.lock"); + write_missing(state / "config.toml", default_config()); + write_missing(state / "ignore", default_ignore); +} +std::size_t clean_workspace(const fs::path& root, bool recursive) { + const auto absolute = checked_root(root); + auto ancestors = cleanup_ancestors(absolute); return clean_directory(*ancestors.back(), recursive); } +WorkspaceStateSnapshot snapshot_workspace_state(const fs::path& root) { + const auto absolute = checked_root(root); + const auto state = absolute / ".same"; + if (!state_exists(state)) + throw std::runtime_error("source .same vanished before move snapshot"); + return state_snapshot(state); +} +void clean_workspace_if_unchanged(const fs::path& root, const WorkspaceStateSnapshot& expected) { + const auto absolute = checked_root(root); + auto ancestors = cleanup_ancestors(absolute); + if (clean_directory(*ancestors.back(), false, &expected) != 1) + throw std::runtime_error("source .same vanished before --move"); +} } // namespace same diff --git a/tests/igpu_routing_tests.cpp b/tests/igpu_routing_tests.cpp index 8aa9197..f739f75 100644 --- a/tests/igpu_routing_tests.cpp +++ b/tests/igpu_routing_tests.cpp @@ -61,6 +61,37 @@ same::Config config() { c.queue_capacity = 8; return c; } +/// 训练请求绕过大小偏好和冷启动信用,但只由指定后端生成标签。 +/// Explicit training bypasses size preference and cold credit without relabeling fallbacks. +void training_override() { + auto c = config(); + c.backend = "auto"; + c.workers = 1; + c.gpu_min_bytes = 1024 * 1024; + c.cuda_bootstrap_ms = 1000000; + c.igpu_bootstrap_ms = 1000000; + same::Resources pool( + c, [](auto, auto) { return std::make_unique(same::BackendKind::cuda); }, + [](auto, auto) { return std::make_unique(same::BackendKind::igpu); }); + for (const auto kind : + {same::BackendKind::cpu, same::BackendKind::cuda, same::BackendKind::igpu}) { + const auto actual = + pool.submit_training_hash( + [kind](same::Worker& worker) { + require(worker.compute->kind() == kind, + "training selected the wrong backend"); + worker.sample = {2048, 1, kind == same::BackendKind::cuda, true, kind}; + return worker.compute->kind(); + }, + 2048, kind) + .get(); + require(actual == kind, "training backend request was ignored"); + } + pool.wait_idle(); + const auto snapshot = pool.profile_snapshot(); + require(snapshot.cpu_samples == 1 && snapshot.gpu_samples == 1 && snapshot.igpu_samples == 1, + "training labels leaked across backends"); +} /// 忙设备不等待,工厂仅调用一次。 / Busy device never waits; initialize exactly once. void admission() { auto c = config(); @@ -399,6 +430,7 @@ int main() { short_scan_discovery(true); short_scan_discovery(false); exploration(); + training_override(); backend_policy(); } catch (const std::exception& e) { std::cerr << e.what() << '\n'; diff --git a/tests/integration_tests.cpp b/tests/integration_tests.cpp index b5a62df..719b1e0 100644 --- a/tests/integration_tests.cpp +++ b/tests/integration_tests.cpp @@ -702,6 +702,7 @@ void command_validation(const fs::path& exe) { } for (const auto& arguments : std::vector>{{"--help"}, {"scan", "--help"}, + {"train", "--help"}, {"new", "--help"}, {"clean", "--help"}, {"--version"}}) { @@ -710,6 +711,23 @@ void command_validation(const fs::path& exe) { check(!read(f.base / "stdout").empty() && !fs::exists(f.root / ".same"), "help/version mutated state or omitted output"); } + // 帮助中的显式颜色可覆盖重定向;自动与 never 保持纯文本。 + // Explicit help color overrides redirection; auto and never stay plain. + for (const auto& arguments : std::vector>{ + {"--color=always", "--help"}, {"--help", "--color=always"}}) { + check(execute(exe, f.root, f.base / "stdout", f.base / "stderr", arguments) == 0, + "colored help failed"); + const auto output = read(f.base / "stdout"); + check(output.find("\033[") != std::string::npos && + output.find("Train options") != std::string::npos && + output.find("--max-files=N") != std::string::npos, + "colored help lost hierarchy or train options"); + } + check(execute(exe, f.root, f.base / "stdout", f.base / "stderr", {"--color=never", "--help"}) == + 0 && + read(f.base / "stdout").find("\033[") == std::string::npos && + !fs::exists(f.root / ".same"), + "plain help emitted ANSI or created state"); } /// 自动模式在设备预算不足时回退;显式环境变量开启真实 GPU 回归。 / Auto mode falls back on @@ -806,8 +824,11 @@ void igpu_backend(const fs::path& exe) { // 新工作区、同一区间至少三个任务;不把两任务区间误当成三设备探索证明。 // A fresh workspace needs three tasks in one band to demonstrate three-device exploration. Fixture automatic(exe); - automatic.config({{"backend", "\"auto\""}, {"workers", "1"}, - {"gpu_min_bytes", "0"}, {"igpu_bootstrap_ms", "0"}, {"cuda_bootstrap_ms", "0"}, + automatic.config({{"backend", "\"auto\""}, + {"workers", "1"}, + {"gpu_min_bytes", "0"}, + {"igpu_bootstrap_ms", "0"}, + {"cuda_bootstrap_ms", "0"}, {"memory_bytes", "134217728"}, {"device_memory_bytes", "134217728"}}); automatic.file("a", content); diff --git a/tests/merge_tests.cpp b/tests/merge_tests.cpp new file mode 100644 index 0000000..0b21484 --- /dev/null +++ b/tests/merge_tests.cpp @@ -0,0 +1,370 @@ +/** @file + * 工作区合并的来源替换、缓存范围与安全删除回归。 + * Workspace merge regressions for source replacement, cache scope and safe cleanup. + */ +#include "same/application.hpp" +#include "same/config.hpp" +#include "same/merge.hpp" +#include "same/model_store.hpp" +#include "same/run_lock.hpp" +#include "same/workspace.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; + +/// Release 构建也保留断言。 / Assertions remain active in Release. +void require(bool okay, const char* message) { + if (!okay) + throw std::runtime_error(message); +} + +/// 测试文件永远位于仓库 .temp。 / Keep all fixtures beneath repository .temp. +fs::path fixture() { + const auto tick = std::chrono::steady_clock::now().time_since_epoch().count(); + return fs::path(SAME_TEST_ROOT) / ".temp" / ("merge-test-" + std::to_string(tick)); +} + +/// 创建内容不同而长度相同的样本。 / Create same-size samples with distinct content. +void file(const fs::path& path, char value) { + std::ofstream out(path, std::ios::binary); + out << std::string(8192, value); + require(bool(out), "cannot create merge fixture file"); +} + +/// 本轮扫描至少形成一个 CPU 模型观测和一个遥测运行。 +/// One scan creates a CPU model observation and a telemetry run. +void scan(const fs::path& root, bool rehash = false) { + same::Config config; + config.workers = 1; + config.metadata_workers = 1; + config.queue_capacity = 4; + config.gpu_min_bytes = 1; + config.backend = "cpu"; + config.rehash = rehash; + std::ostringstream out, err; + same::OutputOptions options; + options.summary = false; + options.recursive = false; + require(same::run(root, config, out, err, options) == 0, "fixture scan failed"); + require(out.str().empty(), "single-file scan invented duplicates"); +} + +/// 精确查询来源账本与聚合样本数。 / Inspect provenance and aggregate sample counts. +std::uint64_t number(const fs::path& path, const char* sql) { + sqlite3* db{}; + const auto name = path.u8string(); + require(sqlite3_open_v2(reinterpret_cast(name.c_str()), &db, SQLITE_OPEN_READONLY, + nullptr) == SQLITE_OK, + "cannot inspect merge database"); + sqlite3_stmt* row{}; + require(sqlite3_prepare_v2(db, sql, -1, &row, nullptr) == SQLITE_OK, + "cannot prepare merge query"); + require(sqlite3_step(row) == SQLITE_ROW, "merge query returned no row"); + const auto value = static_cast(sqlite3_column_int64(row, 0)); + sqlite3_finalize(row); + sqlite3_close(db); + return value; +} + +/// 固定模型载荷中 CPU 样本数位于首块末尾,按小端读取。 +/// CPU sample count occupies the end of the first fixed-format backend block. +std::uint64_t model_samples(const fs::path& path) { + sqlite3* db{}; + const auto name = path.u8string(); + require(sqlite3_open_v2(reinterpret_cast(name.c_str()), &db, SQLITE_OPEN_READONLY, + nullptr) == SQLITE_OK, + "cannot inspect aggregate model"); + sqlite3_stmt* row{}; + require(sqlite3_prepare_v2(db, "SELECT payload FROM models LIMIT 1", -1, &row, nullptr) == + SQLITE_OK, + "cannot query aggregate model"); + require(sqlite3_step(row) == SQLITE_ROW && sqlite3_column_bytes(row, 0) == 744, + "invalid aggregate model payload"); + const auto* bytes = static_cast(sqlite3_column_blob(row, 0)); + std::uint64_t samples = 0; + for (unsigned i = 0; i < 8; ++i) + samples |= std::uint64_t(bytes[240 + i]) << (8 * i); + sqlite3_finalize(row); + sqlite3_close(db); + return samples; +} + +/// 验证重复合并不会放大同一来源,但较新的来源贡献会替换旧值。 +/// Repeated import is idempotent, while a newer source revision replaces old evidence. +void merge_lifecycle() { + const auto base = fixture(); + const auto source = base / "source"; + const auto target = base / "target"; + fs::create_directories(source / ".same"); + fs::create_directories(target / ".same"); + try { + file(source / "a.bin", 'a'); + file(target / "b.bin", 'b'); + { + std::ofstream(source / ".same" / "ignore") << "# source rule\n*.tmp\n"; + std::ofstream(target / ".same" / "config.toml") << "# keep target config\n"; + } + scan(source); + scan(target); + std::ostringstream report; + same::MergeOptions options; + options.summary = true; + require(same::merge_workspaces(target, source, options, report) == 0, "first merge failed"); + require(report.str().find("merge_telemetry_runs=1") != std::string::npos, + "first merge omitted telemetry"); + const auto model = target / ".same" / "model.db"; + const auto state = target / ".same" / "state.db"; + const auto telemetry = target / ".same" / "telemetry.db"; + require(number(model, "SELECT count(*) FROM model_parts") == 2, + "model source ledger omitted a workspace"); + require(model_samples(model) == 2, "first merge lost independent model evidence"); + require(number(state, "SELECT count(*) FROM absolute_cache") == 2, + "absolute cache did not retain both directories"); + require(number(telemetry, "SELECT count(*) FROM runs") == 2, + "telemetry runs were not merged"); + require(fs::exists(target / ".same" / "ignore"), "missing ignore was not copied"); + std::ifstream config(target / ".same" / "config.toml"); + std::string config_line; + std::getline(config, config_line); + require(config_line == "# keep target config", "target configuration was overwritten"); + report.str({}); + report.clear(); + require(same::merge_workspaces(target, source, options, report) == 0, + "repeat merge failed"); + require(number(model, "SELECT count(*) FROM model_parts") == 2 && + number(telemetry, "SELECT count(*) FROM runs") == 2 && + model_samples(model) == 2, + "repeat merge duplicated provenance or telemetry"); + file(source / "a.bin", 'c'); + scan(source); + require(same::merge_workspaces(target, source, options, report) == 0, + "new source revision merge failed"); + require(number(model, "SELECT max(revision) FROM model_parts") == 2 && + number(telemetry, "SELECT count(*) FROM runs") == 3 && + model_samples(model) == 3, + "new source contribution or run not imported"); + scan(target, true); + require(model_samples(model) == 4, + "target-local update discarded an imported model contribution"); + require(number(state, "SELECT count(*) FROM files") == 1, + "external cache entered current scan result scope"); + { + same::RunLock held(source / ".same" / "run.lock"); + options.move = true; + bool locked = false; + try { + same::merge_workspaces(target, source, options, report); + } catch (const std::exception&) { + locked = true; + } + require(locked && fs::exists(source / ".same"), + "active source lock did not protect --move"); + } + require(same::merge_workspaces(target, source, options, report) == 0, + "guarded move failed"); + require(model_samples(model) == 4, + "same source revision was recounted after target-local learning"); + require(!fs::exists(source / ".same") && fs::exists(source / "a.bin"), + "move removed input content or retained source state"); + scan(source); + options.move = false; + require(same::merge_workspaces(target, source, options, report) == 0, + "recreated source workspace merge failed"); + require(model_samples(model) == 5 && number(model, "SELECT count(*) FROM model_parts") == 3, + "recreated path reused an old model-source identity"); + } catch (const std::exception& error) { + std::cerr << "merge fixture failed: " << error.what() << " at " << base << '\n'; + throw; + } + fs::remove_all(base); +} + +/// 递归移动两个来源,且快照变化时不删除源工作区。 +/// Recursive move handles two workspaces, and changed snapshots prevent cleanup. +void recursive_and_changed_source() { + const auto base = fixture(); + const auto parent = base / "parent"; + const auto child = parent / "child"; + const auto target = base / "target"; + fs::create_directories(parent / ".same"); + fs::create_directories(child / ".same"); + fs::create_directories(target); + try { + file(parent / "parent.bin", 'p'); + file(child / "child.bin", 'q'); + scan(parent); + scan(child); + std::ostringstream report; + same::MergeOptions options; + options.recursive = true; + options.move = true; + options.summary = true; + require(same::merge_workspaces(target, parent, options, report) == 0, + "recursive move failed"); + require(report.str().find("merge_workspaces=2") != std::string::npos && + report.str().find("merge_sources_removed=2") != std::string::npos, + "recursive merge omitted a source"); + require(!fs::exists(parent / ".same") && !fs::exists(child / ".same") && + fs::exists(parent / "parent.bin") && fs::exists(child / "child.bin"), + "recursive move removed user content or retained imported state"); + fs::create_directory(parent / ".same"); + std::ofstream(parent / ".same" / "ignore") << "initial\n"; + same::WorkspaceStateSnapshot before; + { + same::RunLock lock(parent / ".same" / "run.lock"); + before = same::snapshot_workspace_state(parent); + } + std::ofstream(parent / ".same" / "ignore") << "changed\n"; + bool refused = false; + try { + same::clean_workspace_if_unchanged(parent, before); + } catch (const std::exception&) { + refused = true; + } + require(refused && fs::exists(parent / ".same"), + "changed source state was removed after snapshot mismatch"); + } catch (const std::exception& error) { + std::cerr << "recursive fixture failed: " << error.what() << " at " << base << '\n'; + throw; + } + fs::remove_all(base); +} + +/// 子来源规则转换成目标根下的限定规则,不误伤其他目录。 +/// Descendant rules are scoped beneath their source relative to the destination. +void scoped_ignore() { + const auto base = fixture(); + const auto source = base / "nested"; + fs::create_directories(source / ".same"); + fs::create_directories(base / ".same"); + try { + std::ofstream(base / ".same" / "ignore") << "# destination\n"; + std::ofstream(source / ".same" / "ignore") << "*.tmp\n!keep.tmp\n/build/\nfoo/bar\n"; + std::ostringstream report; + require(same::merge_workspaces(base, source, {.summary = true}, report) == 0, + "scoped ignore merge failed"); + same::Ignore rules(base); + require(rules.matches("nested/a.tmp", false) && rules.matches("nested/deep/a.tmp", false) && + !rules.matches("other/a.tmp", false) && + !rules.matches("nested/keep.tmp", false) && + rules.matches("nested/build/child", false) && + !rules.matches("nested/deep/build/child", false) && + !rules.matches("other/build/child", false) && + rules.matches("nested/foo/bar", false), + "merged ignore rules lost their descendant scope"); + std::ifstream in(base / ".same" / "ignore"); + const std::string before(std::istreambuf_iterator{in}, {}); + require(same::merge_workspaces(base, source, {.summary = true}, report) == 0, + "repeat ignore merge failed"); + std::ifstream again(base / ".same" / "ignore"); + const std::string after(std::istreambuf_iterator{again}, {}); + require(before == after, "repeat merge duplicated ignore rules"); + } catch (const std::exception& error) { + std::cerr << "ignore fixture failed: " << error.what() << " at " << base << '\n'; + throw; + } + fs::remove_all(base); +} + +/// 父工作区扫描子目录时,导入的绝对缓存免重哈希但结果仍在当前范围。 +/// A parent scan reuses an imported absolute digest without widening its result scope. +void absolute_cache_reuse() { + const auto base = fixture(); + const auto child = base / "child"; + fs::create_directories(child / ".same"); + fs::create_directories(base / ".same"); + try { + file(child / "payload.bin", 'p'); + scan(child); + std::ostringstream report; + require(same::merge_workspaces(base, child, {.summary = true}, report) == 0, + "cannot import child digest archive"); + same::Config config; + config.workers = 1; + config.metadata_workers = 1; + config.backend = "cpu"; + config.pgo = false; + config.telemetry = false; + std::ostringstream out, err; + same::OutputOptions options; + options.recursive = true; + require(same::run(base, config, out, err, options) == 0, + "parent scan after cache import failed"); + require(err.str().find("scanned=1 hashed=0 cached=1") != std::string::npos, + "matching imported absolute digest was not reused"); + require(number(base / ".same" / "state.db", "SELECT count(*) FROM files") == 1, + "cache import widened active scan records"); + } catch (const std::exception& error) { + std::cerr << "cache fixture failed: " << error.what() << " at " << base << '\n'; + throw; + } + fs::remove_all(base); +} + +/// 复制工作区保留旧来源证据,但新的本地训练必须获得新实例身份。 +/// A copied workspace keeps old evidence but forks a fresh local source identity. +void copied_model_identity() { + const auto base = fixture(); + const auto first = base / "first" / ".same"; + const auto copy = base / "copy" / ".same"; + const auto target = base / "target" / ".same"; + fs::create_directories(first); + fs::create_directories(copy); + fs::create_directories(target); + try { + same::detail::OnlineModel model; + require(model.observe(same::BackendKind::cpu, {8192, 4096, 0}, 1.0), + "cannot create model contribution fixture"); + const auto delta = model.delta(); + { + same::ModelStore db(first / "model.db"); + require(db.diagnostic().empty() && db.save("key", delta), + "cannot save first model contribution"); + } + fs::copy_file(first / "model.db", copy / "model.db"); + { + same::ModelStore db(copy / "model.db"); + require(db.diagnostic().empty() && db.save_delta("key", delta, 0.9), + "copied workspace did not fork local model identity"); + require(db.contributions().size() == 2, + "copied workspace reused original contribution identity"); + } + same::ModelStore result(target / "model.db"); + same::ModelStore original(first / "model.db"); + same::ModelStore cloned(copy / "model.db"); + for (const auto& part : original.contributions()) + require(result.merge_contribution(part), "cannot import original contribution"); + for (const auto& part : cloned.contributions()) + require(result.merge_contribution(part), "cannot import copied contribution"); + require(result.contributions().size() == 2 && result.load("key") && + result.load("key")->at(0).samples == 2, + "copy import double-counted original provenance"); + } catch (const std::exception& error) { + std::cerr << "copy fixture failed: " << error.what() << " at " << base << '\n'; + throw; + } + fs::remove_all(base); +} +} // namespace + +int main() { + try { + merge_lifecycle(); + recursive_and_changed_source(); + scoped_ignore(); + absolute_cache_reuse(); + copied_model_identity(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/tests/model_store_tests.cpp b/tests/model_store_tests.cpp index 0b2eb88..9082551 100644 --- a/tests/model_store_tests.cpp +++ b/tests/model_store_tests.cpp @@ -28,7 +28,7 @@ void sql(const std::filesystem::path& path, const char* text) { } // namespace int main() { const auto directory = - std::filesystem::temp_directory_path() / + std::filesystem::path(SAME_TEST_ROOT) / ".temp" / ("same-model-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); try { @@ -54,6 +54,22 @@ int main() { require(!db.save("machine-a", invalid), "NaN accepted"); require(!db.save(std::string(4097, 'x'), state), "oversized key accepted"); } + // 把真实序列化载荷降成旧 schema 2,核实迁移保留模型与初始化历史。 + // Downgrade a real serialized payload to schema 2 and verify migration preserves it. + const auto legacy2 = std::filesystem::canonical(directory) / "legacy2.db"; + std::filesystem::copy_file(path, legacy2); + sql(legacy2, "DROP TABLE model_parts; DROP TABLE setup_parts; DROP TABLE model_meta; " + "PRAGMA user_version=2"); + { + same::ModelStore db(legacy2); + require(db.diagnostic().empty(), "v2 migration failed"); + require(db.load("machine-a") && db.load("machine-a")->at(0).samples == 1, + "v2 migration lost statistics"); + require(db.load_setup("machine-a") && db.load_setup("machine-a")->last_ms == 2, + "v2 migration lost setup history"); + require(db.contributions().size() == 1 && db.setup_contributions().size() == 1, + "v2 migration did not create provenance"); + } { same::ModelStore db(path); require(db.load("machine-a").has_value(), "reopen state missing"); diff --git a/tests/training_tests.cpp b/tests/training_tests.cpp new file mode 100644 index 0000000..a35872f --- /dev/null +++ b/tests/training_tests.cpp @@ -0,0 +1,201 @@ +/** @file + * @brief 独立训练的失败原子性与文件边界。 / Independent training failure and file boundaries. + */ +#include "same/application.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; + +/// 独立于 Release 断言设置,保留具体失败原因。 / Assertions retain failures in Release. +void require(bool condition, const char* message) { + if (!condition) + throw std::runtime_error(message); +} + +/// 只在仓库根目录 .temp 下创建有界语料。 / Keep fixtures beneath the repository .temp. +fs::path fixture_root() { + const auto tick = std::chrono::steady_clock::now().time_since_epoch().count(); + return fs::path(SAME_TEST_ROOT) / ".temp" / ("training-test-" + std::to_string(tick)); +} + +/// 零设备配额使三后端训练确定失败,不依赖测试机器 GPU。 +/// A one-byte device budget makes three-backend training fail independently of hardware. +void unavailable_backend() { + const auto base = fixture_root(); + const auto workspace = base / "workspace"; + const auto corpus = base / "corpus"; + fs::create_directories(workspace); + fs::create_directories(corpus); + try { + std::array bytes{}; + for (unsigned i = 0; i < 4; ++i) { + bytes[i] = static_cast(i + 1); + std::ofstream file(corpus / ("sample-" + std::to_string(i)), std::ios::binary); + file.write(bytes.data(), static_cast(1024 * (i + 1))); + require(bool(file), "cannot create training fixture"); + } + same::Config config; + config.workers = 1; + config.metadata_workers = 1; + config.queue_capacity = 4; + config.gpu_min_bytes = 1; + config.device_memory_bytes = 1; + std::ostringstream output, diagnostics; + bool failed = false; + try { + same::train(workspace, corpus, config, output, diagnostics, 4, 32768); + } catch (const std::runtime_error& error) { + failed = + std::string(error.what()).find("training backend unavailable") != std::string::npos; + } + require(failed, "unavailable accelerator was accepted as a training label"); + require(output.str().empty(), "failed training announced success"); + require(!fs::exists(workspace / ".same" / "state.db"), + "training modified the scan digest cache"); + sqlite3* db = nullptr; + const auto database = workspace / ".same" / "model.db"; + require(sqlite3_open_v2(database.string().c_str(), &db, SQLITE_OPEN_READONLY, nullptr) == + SQLITE_OK, + "cannot inspect model database"); + sqlite3_stmt* query = nullptr; + require(sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM models", -1, &query, nullptr) == + SQLITE_OK, + "cannot inspect model rows"); + require(sqlite3_step(query) == SQLITE_ROW && sqlite3_column_int(query, 0) == 0, + "failed training persisted partial model data"); + sqlite3_finalize(query); + sqlite3_close(db); + const auto telemetry = workspace / ".same" / "telemetry.db"; + require(sqlite3_open_v2(telemetry.string().c_str(), &db, SQLITE_OPEN_READONLY, nullptr) == + SQLITE_OK, + "training failure did not write telemetry"); + require(sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM runs WHERE command='train' AND " + "status='failed'", + -1, &query, nullptr) == SQLITE_OK, + "cannot inspect training run status"); + require(sqlite3_step(query) == SQLITE_ROW && sqlite3_column_int(query, 0) == 1, + "failed training run was not finalized"); + sqlite3_finalize(query); + require(sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM metrics WHERE " + "name='train.model_saved_keys' AND value=0", + -1, &query, nullptr) == SQLITE_OK, + "cannot inspect training final metrics"); + require(sqlite3_step(query) == SQLITE_ROW && sqlite3_column_int(query, 0) == 1, + "failure lost its complete final metrics"); + sqlite3_finalize(query); + require(sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM events WHERE name='train.measure' AND " + "severity='error'", + -1, &query, nullptr) == SQLITE_OK, + "cannot inspect failed training stage"); + require(sqlite3_step(query) == SQLITE_ROW && sqlite3_column_int(query, 0) == 1, + "failure lost its measure-stage error span"); + sqlite3_finalize(query); + require(sqlite3_prepare_v2(db, + "SELECT COUNT(*) FROM logs WHERE name='train.failed' AND " + "severity='error'", + -1, &query, nullptr) == SQLITE_OK, + "cannot inspect training failure log"); + require(sqlite3_step(query) == SQLITE_ROW && sqlite3_column_int(query, 0) == 1, + "failure lost its terminal error log"); + sqlite3_finalize(query); + sqlite3_close(db); + const auto quiet_workspace = base / "quiet-workspace"; + fs::create_directories(quiet_workspace); + config.telemetry = false; + bool quiet_failed = false; + try { + same::train(quiet_workspace, corpus, config, output, diagnostics, 4, 32768); + } catch (const std::runtime_error&) { + quiet_failed = true; + } + require(quiet_failed, "no-telemetry changed training failure semantics"); + require(!fs::exists(quiet_workspace / ".same" / "telemetry.db"), + "disabled training telemetry touched its database"); + for (unsigned i = 0; i < 4; ++i) + require(fs::file_size(corpus / ("sample-" + std::to_string(i))) == 1024 * (i + 1), + "training altered corpus files"); + } catch (...) { + fs::remove_all(base); + throw; + } + fs::remove_all(base); +} + +/// 两目录共用一个选样上限,且重叠目录在创建工作区前拒绝。 +/// Multiple corpora share one selection ceiling; overlap fails before workspace side effects. +void multiple_corpora() { + const auto base = fixture_root(); + const auto workspace = base / "workspace"; + const auto first = base / "first"; + const auto second = base / "second"; + fs::create_directories(workspace); + fs::create_directories(first / "child"); + fs::create_directories(second); + try { + for (const auto& root : {first, second}) + for (int i = 0; i < 2; ++i) { + std::ofstream out(root / ("sample-" + std::to_string(i)), std::ios::binary); + out << std::string(4096 + i * 1024, static_cast('a' + i)); + } + same::Config config; + config.workers = 1; + config.metadata_workers = 1; + config.queue_capacity = 4; + config.gpu_min_bytes = 1; + config.device_memory_bytes = 1; + std::ostringstream output, diagnostics; + bool overlap = false; + try { + same::train(workspace, std::vector{first, first / "child"}, config, output, + diagnostics, 4, 32768, {}); + } catch (const std::runtime_error& error) { + overlap = std::string(error.what()).find("overlap") != std::string::npos; + } + require(overlap && !fs::exists(workspace / ".same"), + "overlapping corpora mutated workspace"); + bool failed_device = false; + try { + same::train(workspace, std::vector{first, second}, config, output, + diagnostics, 4, 32768, {}); + } catch (const std::runtime_error& error) { + failed_device = + std::string(error.what()).find("training backend unavailable") != std::string::npos; + } + require(failed_device, "multi-corpus selection did not reach backend validation"); + sqlite3* db{}; + const auto telemetry = workspace / ".same" / "telemetry.db"; + require(sqlite3_open_v2(telemetry.string().c_str(), &db, SQLITE_OPEN_READONLY, nullptr) == + SQLITE_OK, + "cannot inspect multi-corpus telemetry"); + sqlite3_stmt* row{}; + require(sqlite3_prepare_v2(db, + "SELECT value FROM metrics WHERE name='train.selected_files'", + -1, &row, nullptr) == SQLITE_OK, + "cannot inspect global selection count"); + require(sqlite3_step(row) == SQLITE_ROW && sqlite3_column_int(row, 0) == 4, + "multi-corpus training did not select from both directories"); + sqlite3_finalize(row); + sqlite3_close(db); + } catch (...) { + fs::remove_all(base); + throw; + } + fs::remove_all(base); +} +} // namespace + +int main() { + unavailable_backend(); + multiple_corpora(); +} From 9b9601881f302245697a94159d8c0ce8ceeb1252 Mon Sep 17 00:00:00 2001 From: moeSegFault Date: Sun, 27 Sep 2026 16:24:02 +0800 Subject: [PATCH 2/2] chore(release): prepare v0.6.0 distribution Bump the CMake product version, update the public release page, and add v0.6.0 notes with feature scope and limitations. Keep packaging smoke fixtures under repository .temp. Verified the versioned Windows CUDA Release build, tag/version check, and staged/extracted package smoke; cross-platform package validation is the next GitHub Actions gate. --- CMakeLists.txt | 2 +- docs/releases/v0.6.0.md | 25 +++++++++++++++++++++++++ site/index.html | 8 ++++---- tools/package_release.py | 11 +++++++++-- 4 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 docs/releases/v0.6.0.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 6aa81af..69cb54a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.25) -project(same VERSION 0.5.1 LANGUAGES C CXX) +project(same VERSION 0.6.0 LANGUAGES C CXX) # Native language/target metadata is shared by builds and IDE code models. # 原生语言及目标元数据同时服务构建与 IDE 代码模型。 diff --git a/docs/releases/v0.6.0.md b/docs/releases/v0.6.0.md new file mode 100644 index 0000000..9286e32 --- /dev/null +++ b/docs/releases/v0.6.0.md @@ -0,0 +1,25 @@ +# same v0.6.0 + +## 下载 / Downloads + +- Windows x64 and Linux x64: `standard` includes CPU and optional OpenCL iGPU; `cuda` additionally includes CUDA. +- macOS Apple Silicon: `standard` includes CPU and optional OpenCL iGPU; CUDA is unavailable. +- CUDA packages require a compatible NVIDIA driver, not a local CUDA compiler. A GPU is not required for ordinary CPU scanning. +- Windows targets Windows 10/11 x64; Linux packages are built on Ubuntu 24.04 (glibc 2.39+); macOS targets macOS 14+ ARM64 and is not notarized. +- Verify `SHA256SUMS` and build provenance. Archives include licenses and `build-info.json` identifying the exact commit and backend configuration. + +## 更新 / Changes + +- Added `same train [options]`: select a bounded, size-stratified corpus across multiple directories and measure CPU, CUDA and iGPU on each selected file. With no directory, the corpus defaults to `.`; the former `--corpus=DIR` option remains supported. Training updates the **same** online model subsequently used by automatic scans, not a separate training database. Missing or failing accelerators fail training rather than silently recording CPU fallback as GPU data. +- Added end-to-end training telemetry and an opt-in, sectioned `train --summary`. Successful training is quiet without `--summary`; warnings and failures remain visible. Help output also gains color-coded hierarchy in an interactive terminal without changing its plain-text meaning. +- Added `same merge [dir] [-r] [--move] [--summary]` for model contributions, scan cache, telemetry runs and missing ignore rules. The destination configuration remains unchanged. Descendant ignore rules are scoped to their subtree; rules from an unrelated directory apply at the destination root with a warning. Guarded `--move` removes only source `.same` state after successful import and an unchanged-state check. +- Model provenance makes remerging a source idempotent: when that source trains again, its newer contribution replaces its older one rather than double-counting history. Existing model databases migrate to the new schema. Scan cache gains absolute-path archival while ordinary duplicate results remain limited to the current scan scope; an imported digest is reused only after a complete file-stamp match, and exact duplicates still require bytewise verification. +- Telemetry runs are imported by run ID with dependent events, metrics and parameters. A repeated merge skips already imported runs. Imported telemetry remains subject to the destination's later retention policy. + +## 验证与边界 / Validation and limits + +Before tagging, local Windows CPU Debug CTest passed 27/27. An isolated Windows Release smoke used physical CUDA and iGPU backends to train three files from two corpus directories: each backend returned three verified digest samples, and all nine accepted training traces persisted. The resulting model and telemetry were imported into a fresh workspace using the Release `merge` command. This is functional evidence, not a controlled scan-speedup benchmark. Hosted GitHub Actions rebuilds and tests the five release packages across Windows, Linux and macOS before publishing; hosted runners without a physical GPU cannot establish physical GPU performance. + +`train` requires all three backends and representative input; it is not useful on a CPU-only installation. Cold accelerator setup can exceed the cost of short tasks, so normal automatic scans retain their cautious startup policy. Cache stamps are not an adversarial filesystem snapshot. `merge` is independently transactional and retryable per database, **not one atomic transaction across all databases**; `--move` is not globally atomic across multiple sources. Review external ignore rules before scanning. Training calibration and model prediction error do not by themselves prove an end-to-end speedup. + +Scanning never deletes input files and remains nonrecursive by default. Use `same scan -r --summary` to opt into recursion and inspect diagnostics. diff --git a/site/index.html b/site/index.html index d262de8..48ca3c1 100644 --- a/site/index.html +++ b/site/index.html @@ -1,9 +1,9 @@ -same — 找到真正相同的文件 +same — 找到真正相同的文件
same / MoeSegfault
-

LOCAL FIRST · EXACT BYTES · v0.5.1

相同的文件,
不必靠猜。

先快速筛选,再逐字节确认。
把重复找出来,把决定留给你。

C++23 / Windows · Linux · macOS / GPL-3.0

YOUR FILES. YOUR MACHINE.
# 只检查当前目录
+

LOCAL FIRST · EXACT BYTES · v0.6.0

相同的文件,
不必靠猜。

先快速筛选,再逐字节确认。
把重复找出来,把决定留给你。

C++23 / Windows · Linux · macOS / GPL-3.0

YOUR FILES. YOUR MACHINE.
# 只检查当前目录
 $ same
 
 [SAME] Group 1
@@ -13,9 +13,9 @@
 # 想看子目录?显式开启。
 $ same scan -r --summary

扫描不会删除或改写待扫描文件。

01 / CORRECTNESS

摘要只是筛选

BLAKE3 找出候选,逐字节比较确认相同。不是“哈希一样,所以一样”。

02 / CONTROL

范围由你定义

默认不进入子目录。Gitignore 语法的忽略规则、显式递归,让扫描范围清晰可见。

03 / PERFORMANCE

让硬件做合适的事

并行元数据与内容处理,复用缓存;CPU SIMD 默认可用,线程私有模型根据真实任务成本选择 CPU、CUDA 或 iGPU,并在启动设备前检查开销是否值得。

-

DOWNLOAD · VERIFY · RUN

下载即用,能力说清楚。

选择发布包

下载 GitHub Release ↗

  • Windows x64 / Linux x64:standard 包含 CPU 与可选 iGPU;cuda 包额外包含 CUDA 后端。
  • macOS Apple Silicon:standard 包,CPU 与可选 iGPU;不支持 CUDA。
  • CUDA 包需要兼容的 NVIDIA 驱动,不需要自行安装编译器。设备不可用时保留 CPU 路径。

每个包均由 Actions 构建、测试并验证解包后的程序,附 SHA-256 校验、来源证明与准确构建信息。Linux 基线为 Ubuntu 24.04;macOS 为 14+ ARM64,尚未公证。

发布与平台要求 ↗

也可以从源码构建

git clone https://github.com/kleedaisuki/same.git
+

DOWNLOAD · VERIFY · RUN

下载即用,能力说清楚。

选择发布包

下载 GitHub Release ↗

  • Windows x64 / Linux x64:standard 包含 CPU 与可选 iGPU;cuda 包额外包含 CUDA 后端。
  • macOS Apple Silicon:standard 包,CPU 与可选 iGPU;不支持 CUDA。
  • CUDA 包需要兼容的 NVIDIA 驱动,不需要自行安装编译器。设备不可用时保留 CPU 路径。

每个包均由 Actions 构建、测试并验证解包后的程序,附 SHA-256 校验、来源证明与准确构建信息。Linux 基线为 Ubuntu 24.04;macOS 为 14+ ARM64,尚未公证。

阅读 v0.6.0 发布说明 ↗ · 发布与平台要求 ↗

也可以从源码构建

git clone https://github.com/kleedaisuki/same.git
 cd same
 cmake --preset release
-cmake --build --preset release

需要 CMake 3.25+、C++23 编译器;Windows 使用 VS x64 开发者终端。

完整构建指南 ↗

日常命令

same new
生成默认配置与 ignore,不覆盖已有文件。
same scan -r --summary
递归扫描并查看结果、设备决策及学习统计;省略 -r 不进入子目录。
same scan --cpu
明确使用 CPU;--cuda 和 --igpu 可用于设备对照。
same scan --no-telemetry
关闭遥测日志,仍保留在线学习;--no-pgo 才关闭学习及模型读写。
same clean [-r]
删除 .same 状态,包括配置、ignore、摘要缓存、模型和遥测;请先备份。

RELEASE NOTES / 0.5.1

看清结果,也选对工具链。

完整保留本轮诊断,用清晰分区和彩色语义快速阅读;Windows 构建按真实 CUDA 能力选择兼容工具链。

汇总布局与颜色语义 ↗
构建与 CUDA 工具链 ↗

  • 八个分区,完整证据:结果、存储、性能、路由、在线模型、工作线程、模型持久化与遥测依次展开,不省略原有统计。
  • 颜色只做加速:成功、警告、错误、计时和模型证据具有语义高亮;关闭颜色后,标签与状态词仍表达全部含义。
  • 机器接口不动:TSV、原始 key=value、stdout/stderr、退出码及 NO_COLOR 契约保持兼容。
  • CUDA 选择有证据:Windows 从新到旧探测 VS x64 环境,只有能与所选 nvcc 完成 C++20 编译链接的实例才会被采用。
  • 严格发布门:五类跨平台包分别构建、测试、解包烟雾验证并生成校验和;标签与版本不一致时拒绝发布。
+cmake --build --preset release

需要 CMake 3.25+、C++23 编译器;Windows 使用 VS x64 开发者终端。

完整构建指南 ↗

日常命令

same new
生成默认配置与 ignore,不覆盖已有文件。
same scan -r --summary
递归扫描并查看结果、设备决策及学习统计;省略 -r 不进入子目录。
same train ../photos ../archive --summary
在多个目录选择训练文件,分别测量 CPU、CUDA 与 iGPU;结果写入当前工作区的在线模型。需要三个设备后端均可用。
same merge ../other --summary
将其他工作区的模型贡献、缓存和遥测合并到当前工作区,保留本地配置。
same scan --no-telemetry
关闭遥测日志,仍保留在线学习;--no-pgo 才关闭学习及模型读写。
same clean [-r]
删除 .same 状态,包括配置、ignore、摘要缓存、模型和遥测;请先备份。

RELEASE NOTES / 0.6.0

让模型有数据,也让经验能迁移。

显式训练在真实文件上同时测量三个后端;工作区合并汇集模型、摘要缓存与遥测,不重复计入同一来源。

查看完整 v0.6.0 发布说明 ↗
了解合并语义与边界 ↗

  • 训练即使用:same train <dirs> --summary 以统一预算选取多目录语料;成功样本直接更新普通自动扫描所加载的模型,不另存训练专用模型。
  • 观测完整且易读:训练记录阶段与各后端尝试;--summary 分区展示样本、耗时、模型和遥测,交互终端自动配色,机器重定向保持可解析。
  • 跨工作区合并:same merge [dir] -r --summary 导入模型来源贡献、绝对路径缓存、遥测与缺失的 ignore 规则;保留目标配置。
  • 重复同步不重复学习:来源再训练后重新合并,会替换其较旧模型贡献;扫描结果仍仅包含当前扫描范围,并继续逐字节确认重复。
  • 谨慎移动与发布:--move 只在导入成功并核对来源未变化后删除源 .same;五类发行包仍须经跨平台 Actions 构建、测试和解包验证。
diff --git a/tools/package_release.py b/tools/package_release.py index c727a69..f0ed77d 100644 --- a/tools/package_release.py +++ b/tools/package_release.py @@ -18,6 +18,13 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] +TEMP_ROOT = ROOT / ".temp" + + +def temporary_workspace(prefix): + """将发布测试文件限制在仓库内。 / Keep release-test files inside the repository.""" + TEMP_ROOT.mkdir(exist_ok=True) + return tempfile.TemporaryDirectory(prefix=prefix, dir=TEMP_ROOT) def version(): @@ -46,7 +53,7 @@ def smoke(binary, expected_version): """检查发布二进制的版本和精确分组;无需 GPU。 / Check packaged version and exact groups without a GPU.""" if run([str(binary), "--version"]).strip() != "same " + expected_version: raise RuntimeError("binary/version mismatch") - with tempfile.TemporaryDirectory(prefix="same-package-smoke-") as directory: + with temporary_workspace("same-package-smoke-") as directory: root = Path(directory).resolve() (root / "a").write_bytes(b"same release smoke\n") (root / "b").write_bytes(b"same release smoke\n") @@ -86,7 +93,7 @@ def dependency(name): name = f"same-v{version()}-{args.platform}-{args.flavor}" output = Path(args.output).resolve() output.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix="same-package-") as directory: + with temporary_workspace("same-package-") as directory: stage = Path(directory) / name (stage / "bin").mkdir(parents=True) shutil.copy2(binary, stage / "bin" / exe)