diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca8a112..8b49c92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,8 @@ jobs: test: runs-on: windows-latest timeout-minutes: 20 + env: + SCOUT_RETAIN_FIXTURES: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a56ded8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,19 @@ +# Model Scout + +Start with [HANDOFF.md](HANDOFF.md), then read only the linked guidance relevant +to the task. Inspect Git status and recent commits before editing. Preserve WIP. + +Keep personal libraries and live app profiles untouched during development. +Use generated fixtures and isolated `SCOUT_TEST_DATA` profiles. Cloud calls +require explicit opt-in. Archive contents are opt-in for each scan. Transfers +must be reviewed, exclusive and verified; never overwrite originals or targets. + +Never delete files, directories, branches, worktrees or user records without the +owner's explicit approval. Set `SCOUT_RETAIN_FIXTURES=1` for development/CI checks +unless fixture deletion has been authorized. This mode omits destructive fixture +checks; it does not establish source-removal or collection-removal UI coverage. +Keep generated artifacts. Do not reset, clean, stash or broadly stage work. + +Maintain the canonical handoff with scope, changed paths, checks, limitations and +next steps. Keep private catalogs, source paths and logs out of Git. Use explicit +file paths for commits. Match documentation claims to fresh evidence. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc94ba..1d00976 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.5.2 + +- Add game-library scanning of all regular file types, including editor sources, + unknown extensions, documentation and extensionless files; retain model mode. +- Add file-family and extensionless filters, saved family filters, local raster + previews and a unique-results view that respects current filters. +- Prepare reviewed unique-file libraries with fresh SHA-256 hashes, a new + category/family organization, duplicate aliases, historical original paths, + collection provenance, category JSONL indexes and a Markdown entry point. +- Copy unique assets with source/hash checks, exclusive targets, verification, + cancellation and copy journals; keep every original source file. +- Flag maps, atlases and other reference-bearing files for review before game + use. Reference rewriting and source/editor conversion are not included. +- Batch hash persistence and bound index writers for large libraries. Retain + generated build outputs instead of clearing the output directory on each build. +- Split category indexes into 1,000-entry chunks and place unique assets in + hash-prefix subfolders to keep large collections navigable. Sanitize Windows + reserved folder names and package only current manifest outputs. +- Retain duplicate aliases when preparing from the unique-results view, reject + destinations inside any scanned root, and restore catalog mode and roots after + cancelled or interrupted scans. + ## 0.4.1 - Search filenames by default, with explicit path and all-details scopes retained in saved searches, selection, and CSV exports. diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000..576146e --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,83 @@ +# Model Scout current handoff + +## 2026-10-03 — Game asset library + +Goal: extend the existing discovery, filtering, duplicate and copy workflows to +support a complete game-asset directory. The owner will choose the destination +and run the feature in the updated application. Development must not copy, +move or delete personal assets or use the live app profile. + +Approved design: game mode indexes all file extensions and extensionless files; +archive contents remain opt-in. Prepare a new unique-file library with fresh +SHA-256 hashes, category/family folders, readable names, source aliases, +historical original-path mappings when available, category JSONL indexes and a +Markdown entry point. Copying requires a separate reviewed action. Preserve +originals and never overwrite targets. Flag relative-reference files; do not +claim that maps/atlases/projects are repaired or ready for runtime use. + +Branch: `codex/game-asset-library`, based on merged PR #2 / main `92789a0`. +Implementation is complete in `app/game-formats.cjs`, `app/game-library.cjs`, the +scanner, catalog, IPC/preload, asset-library dialog, raster previews and tests. +New asset folders use category/family/hash-prefix paths and readable hashed names; +category JSONL indexes have at most 1,000 entries per chunk. Hash updates are +batched and index writers are bounded. Packaging retains earlier frontend/output +files and selects only current manifest outputs. + +Initial evidence: game discovery and verified unique-copy tests pass. A separate +read-only inventory counted 201,335 files, 609 directories and 60 extensions; +fresh content hashing found 160,096 unique hashes and 41,239 extra copies, with +no read errors, source fingerprint changes or historical hash mismatches. +These private library facts are validation evidence, not distributable content. + +Verification: 31 core/feature tests pass with `SCOUT_RETAIN_FIXTURES=1`; all eight +source and all eight 0.5.2 packaged desktop workflows pass. Format/build, +dependency audit, unsigned Windows packaging and full source/notices ZIP pass. +ZIP SHA-256: `e32f88e4951f7cd4fb4d71729662074fff1fc56d640040aaa3c6f41e8d6afde0`. +Artifact: `release/0.5.2/Model-Scout-0.5.2-Windows-x64.zip`. ASAR verification +matched six core source files and all six manifest frontend outputs; the ZIP +contains 20 entries including four pinned importer/source archives and notices. +The real game-mode scan took 25.09 seconds with zero errors and an average +unfiltered gallery query of 19.42 ms. Private receipt: +`artifacts/benchmarks/1791080840283/receipt.json` (never publish its source paths). + +A negative anchored test-name exclusion unexpectedly ran the earlier full +31-test suite, including generated fixture removals, and all passed. This was +reported to the owner; only isolated fixtures were affected. A harmless probe +verified positive skip-pattern behavior. Retained-fixture checks now omit four +core move/relocation tests and desktop source/collection removal. The owner has +not answered the deletion exception question; do not run those remaining +destructive desktop workflows without approval. CI uses retained-fixture mode. + +Delivery: [PR #3](https://github.com/the3dcoder/ModelScout/pull/3) is open and +attached to the app task. Check its live Windows checks for hosted verification; +the local checks below cover the actual source and packaged build. After review, +the owner can extract the complete ZIP, select Game asset library, scan, choose +Create asset library, prepare, then confirm the reviewed copy. Do not run that +copy on their behalf. Keep local candidates/artifacts. The unique library is a +snapshot at preparation time; it does not continuously synchronize source files. + +Review checkpoint: three P2 findings reproduced with failing tests, then fixed: +unique-view preparation now expands only the display collapse to retain aliases; +destinations are checked against all scanned roots including empty/unmatched ones; +cancelled and interrupted scans restore previous mode/root metadata with attempted +scope retained separately. The 31 retained core tests pass after these fixes. +Read-only follow-up review found no remaining Important/Critical findings. +Final 0.5.2 checks passed: `SCOUT_RETAIN_FIXTURES=1 npm test` (31), build, +`npm run test:app` (eight), `npm run format:check`, `npm audit --omit=dev` (zero), +`npm run package`, retained `npm run test:packaged` (eight), and +`npm run release:bundle`. The unsigned ZIP hash and packaged source/manifest +bytes were independently checked. Preserve earlier local 0.5.0/0.5.1 candidates. + +Hosted checkpoint: paired runs on both initial heads disagree on the cancelled +mode UI check: one passes while one times out waiting for the family selector. +Core/build checks pass. Ten additional local game workflows and a renderer run +at 6x CPU throttling pass. Progress/metadata failure diagnostics were added; +the diagnostic head `c2d0db6` passed both hosted Windows runs +[37173760635](https://github.com/the3dcoder/ModelScout/actions/runs/37173760635) +and [37173762458](https://github.com/the3dcoder/ModelScout/actions/runs/37173762458). +Read-only tracing found no confirmed production race. Keep the diagnostics and +investigate if the earlier intermittent timeout recurs; do not claim its cause +was fixed. No production code changed after the verified 0.5.2 bundle. + +See [README](README.md), [verification](docs/VERIFICATION.md) and +[roadmap](docs/ROADMAP.md) for current product behavior and evidence boundaries. diff --git a/README.md b/README.md index d4d2cb6..b1bcb21 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Model Scout -A local Windows workbench for finding, previewing, and organizing 3D models and printing files. +A local Windows workbench for finding, previewing, and organizing 3D models, printing files, and game asset libraries. Search folders or drives, browse a cached gallery, keep multi-part projects together, and review exactly what will be copied or moved. Model Scout also includes filament and resin cost profiles and basic STL inspection and editing. ## Run it -Use Windows 10/11 x64. Build the release ZIP with the commands below, or download a published build when one is available under [Releases](https://github.com/the3dcoder/ModelScout/releases). Extract the complete ZIP and run **Model-Scout-0.4.1.exe**. No installer is required. The current build is unsigned. +Use Windows 10/11 x64. Build the release ZIP with the commands below, or download a published build when one is available under [Releases](https://github.com/the3dcoder/ModelScout/releases). Extract the complete ZIP and run **Model-Scout-0.5.2.exe**. No installer is required. The current build is unsigned. The portable executable stores its catalog, thumbnails, profiles, and transfer logs in `%APPDATA%\Model Scout`. Back up that folder while the app is closed if you want to preserve your library metadata. Source models stay in their original folders until you confirm a move. @@ -17,7 +17,60 @@ npm ci npm start ``` -## Organize a library +## Create a unique game asset library + +1. Select **Game asset library · all files**, add folders, then **Search locations**. + This mode retains every regular file, including unknown extensions and + extensionless license files. Links/junctions are skipped. Archive contents + remain opt-in; the archive containers themselves are indexed. +2. Browse by family, extension, suggested category, tags or filename. Common + raster images up to 8 MiB have local on-demand previews; 3D thumbnails retain + their existing cache. SVG/editor formats and unsupported media remain listed. +3. Choose **Create asset library** and select the entire scan or current filters. + Choose an existing parent outside the scanned folders. Preparation creates a + new timestamped folder, hashes each selected file, and writes a reviewable + catalog and full copy plan. A failed or cancelled catalog cannot be copied. +4. Review the unique count, size, sample organization, full copy plan and reference + warnings. Confirm the review checkbox, then **Copy unique assets to new library**. + Copies use new category/family/hash-prefix folders and shortened readable names with full + SHA-256 suffixes. Identical category/family levels are combined. Exactly one + copy is planned per SHA-256; every original remains in place. Targets are never + overwritten, source fingerprints/hashes are rechecked, and copies are verified. +5. Give another chat the new folder's **START_HERE.md** path. It links to smaller + category indexes, split into at most 1,000 entries each. `assets.jsonl` contains one canonical asset per hash; + `sources.jsonl` retains every source/duplicate alias, pack mapping, tags and + collection provenance. `copy-plan.jsonl` records full destinations; + `copy-receipt.json` and `copy-journal.jsonl` establish which copies succeeded. + +The **Unique files** view collapses hash-confirmed duplicates within the active +filters. Unhashed files stay visible. Run **Check duplicates** or prepare an asset +library to establish hashes; a filename match alone never removes a result. +Preparing from this view expands the duplicate collapse, preserving every matching +source alias and freshly verifying its content. The remaining file filters apply. + +Categories inferred from source folders and file extensions require review. +When `00_CATALOG/FILE_INDEX.jsonl` exists, bounded historical original-path/pack +mappings are retained; its old hashes are never trusted for duplicate decisions. +Historical mappings are information about earlier organization, not proof that +old source paths still exist. License documents are indexed, but licensing is not +inferred. Preparation, catalog export and copying stay local. + +This is an organized source library. Phaser atlases/fonts and Tiled maps can need +companion files and relative paths, so reference-bearing formats are flagged. +The unique library changes names and paths; it does not rewrite references or +convert source/editor assets into runtime-ready files. Consult the aliases to +reconstruct or repair asset sets before use. See the primary +[Phaser loader documentation](https://docs.phaser.io/phaser/concepts/loader) and +[Tiled map format](https://doc.mapeditor.org/en/stable/reference/json-map-format/). + +Cancellation retains generated files and writes incomplete status; copying stops +between operations and retains originals. A process exit can leave a partial +library; consult the journal and prepare a fresh library to retry. Plans are valid +for 24 hours in the current app session and cannot be executed twice. Keep existing +partial outputs until you decide how to handle them. Review free disk space for +the assets and catalogs before copying. + +## Organize a model library 1. Add search folders or drives. ZIP scanning is optional; 7z and RAR additionally require a local [7-Zip installation](https://www.7-zip.org/). 2. Search filenames by default, without inheriting matches from parent folders or archives. Matching folder and ZIP/7z/RAR names appear separately; their contents are not included automatically. Choose **Paths** or **All file details** to search broader fields explicitly. Combine tags with all/any/exclude filters, mark favorites, and save searches. @@ -62,6 +115,7 @@ Scanning, hashing, thumbnails, costs, and mesh tools run locally. There is no te ## Development and releases ```powershell +$env:SCOUT_RETAIN_FIXTURES = "1" npm test npm run build npm run test:app @@ -73,6 +127,18 @@ npm run release:bundle Tests use generated fixtures and isolated profiles. They do not use a personal model library. The public release ZIP includes the executable, licenses, and corresponding source snapshots for the native importers. Distribute that complete ZIP. See [CONTRIBUTING.md](CONTRIBUTING.md), [verification](docs/VERIFICATION.md), and [importer build instructions](docs/BUILD_IMPORTERS.md). +The retained-fixture mode above omits source-removal/metadata-removal checks and +tests copying in the duplicate desktop flow. CI uses this mode. Only unset it +when deletion of generated test fixtures is authorized. Builds retain earlier +versioned frontend outputs; packaging uses only the current Vite manifest. +An existing unpacked release or release ZIP is preserved: choose a fresh output +before packaging or bundling the same version again. + +The unique library is a snapshot of the selected files at preparation time. +Prepare again after adding or editing source assets; excluded duplicate aliases +are recorded as they existed when hashed. Canonical sources are rechecked during +copying. This feature does not continuously synchronize a changing directory. + Current limits: Windows x64 is the tested target; nested archives are not expanded; preview reads stop at 128 MiB; complex CAD/native application files may require their original software. Transfers are logged but do not have automatic undo. Scans skip symbolic links and junctions inside the search tree. Shape similarity, advanced mesh repair, website synchronization, and direct slicer integrations remain [future work](docs/FEATURE_RESEARCH.md). ## License and credits diff --git a/app/catalog.cjs b/app/catalog.cjs index 7cd079d..17d6004 100644 --- a/app/catalog.cjs +++ b/app/catalog.cjs @@ -113,8 +113,13 @@ class Catalog { beginScan() { this.flush(); this.db.exec( - "BEGIN; DROP TABLE IF EXISTS scan_backup; CREATE TABLE scan_backup AS SELECT * FROM files; DROP TABLE IF EXISTS locations_backup; CREATE TABLE locations_backup AS SELECT * FROM locations; DELETE FROM files; DELETE FROM locations; COMMIT", + "BEGIN; DROP TABLE IF EXISTS scan_backup; CREATE TABLE scan_backup AS SELECT * FROM files; DROP TABLE IF EXISTS locations_backup; CREATE TABLE locations_backup AS SELECT * FROM locations; DELETE FROM files; DELETE FROM locations", ); + this.setting( + "scanPrevious", + this.setting("lastScan") || { mode: "models", roots: [] }, + ); + this.db.exec("COMMIT"); } finishScan(restore = false) { this.flush(); @@ -125,10 +130,22 @@ class Catalog { ) { this.db.exec("BEGIN"); try { - if (restore) + if (restore) { this.db.exec( "DELETE FROM files; INSERT INTO files SELECT * FROM scan_backup", ); + const previous = this.setting("scanPrevious"); + const attempted = this.setting("lastScan"); + if (previous) + this.setting("lastScan", { + ...previous, + attemptedMode: attempted?.mode, + attemptedRoots: attempted?.roots || [], + phase: "cancelled", + running: false, + restoredPrevious: true, + }); + } if ( this.db .prepare( @@ -142,6 +159,7 @@ class Catalog { ); this.db.exec("DROP TABLE locations_backup"); } + this.setting("scanPrevious", null); this.db.exec("DROP TABLE scan_backup; COMMIT"); return true; } catch (e) { @@ -183,7 +201,9 @@ class Catalog { } get(id) { return present( - this.db.prepare("SELECT " + FIELDS + FROM + "WHERE f.id=?").get(id), + (this.getStatement ||= this.db.prepare( + "SELECT " + FIELDS + FROM + "WHERE f.id=?", + )).get(id), ); } setting(key, value) { @@ -233,12 +253,25 @@ class Catalog { } if (ext) { clauses.push("f.ext=?"); - params.push(ext); + params.push(ext === "(none)" ? "" : ext); } if (category) { clauses.push("COALESCE(n.category,f.category)=?"); params.push(category); } + if (options.family) { + clauses.push("f.family=?"); + params.push(String(options.family)); + } + if (kind === "unique") { + const scope = this.filters({ ...options, kind: "" }); + clauses.push( + "(f.hash IS NULL OR f.id IN (SELECT MIN(f.id) FROM files f LEFT JOIN notes n USING(id)" + + scope.where + + " GROUP BY f.hash))", + ); + params.push(...scope.params); + } if (kind === "archive") clauses.push("f.member<>''"); if (kind === "loose") clauses.push("f.member=''"); if (kind === "favorites") clauses.push("n.favorite=1"); @@ -360,6 +393,11 @@ class Catalog { "SELECT ext,COUNT(*) AS count FROM files GROUP BY ext ORDER BY count DESC", ) .all(), + families: this.db + .prepare( + "SELECT family,COUNT(*) AS count FROM files GROUP BY family ORDER BY family", + ) + .all(), categories: this.db .prepare( "SELECT COALESCE(n.category,f.category) AS category,COUNT(*) AS count FROM files f LEFT JOIN notes n USING(id) GROUP BY COALESCE(n.category,f.category) ORDER BY category", @@ -395,6 +433,24 @@ class Catalog { .all(...params) .map((r) => r.id); } + *iterateFiles(options = {}) { + const { where, params } = this.filters(options); + const statement = this.db.prepare( + "SELECT " + + FIELDS + + FROM + + where + + (where ? " AND " : " WHERE ") + + "f.id>? ORDER BY f.id LIMIT 512", + ); + let after = ""; + for (;;) { + const rows = statement.all(...params, after); + if (!rows.length) return; + after = rows[rows.length - 1].id; + for (const row of rows) yield present(row); + } + } queryGeometryRows() { return this.db .prepare( @@ -478,6 +534,26 @@ class Catalog { this.db.prepare("UPDATE files SET hash=? WHERE id=?").run(hash, id); this.db.prepare("UPDATE asset_cache SET hash=? WHERE id=?").run(hash, id); } + setHashes(entries) { + if (!entries.length) return; + this.flush(); + const file = this.db.prepare("UPDATE files SET hash=? WHERE id=?"), + cache = this.db.prepare("UPDATE asset_cache SET hash=? WHERE id=?"); + this.db.exec("BEGIN"); + try { + for (const { id, hash } of entries) { + const row = this.get(id); + if (!row) continue; + this.ensureCache(row); + file.run(hash, id); + cache.run(hash, id); + } + this.db.exec("COMMIT"); + } catch (e) { + this.db.exec("ROLLBACK"); + throw e; + } + } thumbnail(id) { return this.db.prepare("SELECT c.thumbnail" + FROM + "WHERE f.id=?").get(id) ?.thumbnail; @@ -538,6 +614,7 @@ class Catalog { "searchScope", "ext", "category", + "family", "kind", "sort", "direction", diff --git a/app/formats.cjs b/app/formats.cjs index 3a6b965..ff4b53b 100644 --- a/app/formats.cjs +++ b/app/formats.cjs @@ -132,15 +132,21 @@ function suggest(name, folder = "") { confidence: "Unknown", }; } -function fileRecord(filePath, stat, root, member = "") { +function fileRecord(filePath, stat, root, member = "", mode = "models") { const name = member ? path.posix.basename(member.replaceAll("\\", "/")) : path.basename(filePath); const ext = path.extname(name).slice(1).toLowerCase(); - const guessed = suggest( - name, - member ? path.posix.dirname(member) : path.dirname(filePath), - ); + const guessed = + mode === "game" + ? require("./game-formats.cjs").gameDetails( + name, + member || path.relative(root, filePath), + ) + : suggest( + name, + member ? path.posix.dirname(member) : path.dirname(filePath), + ); return { path: filePath, member, diff --git a/app/game-formats.cjs b/app/game-formats.cjs new file mode 100644 index 0000000..f59d747 --- /dev/null +++ b/app/game-formats.cjs @@ -0,0 +1,77 @@ +const path = require("node:path"); +const GROUPS = [ + [ + "Images & textures", + "png jpg jpeg webp gif bmp tga tif tiff avif ico ktx ktx2 pvr dds basis", + ], + ["Vector graphics", "svg eps"], + ["Audio", "ogg oga wav mp3 m4a aac flac opus mid midi"], + ["Fonts", "ttf otf woff woff2 eot fnt"], + ["Maps & atlases", "tmx tsx tmj tsj world atlas ldtk ldl spriteatlas"], + [ + "Data & configuration", + "json jsonl xml csv yaml yml ini toml bin meta mat asset anim controller mcmeta pal", + ], + ["Shaders", "glsl frag vert wgsl shader hlsl fx"], + ["Code & web", "js mjs cjs ts jsx html htm css scss less py lua cs"], + [ + "Editor projects", + "ase aseprite psd psb kra ai xcf afphoto afdesign capx c3p unitypackage blend max f3d fcstd", + ], + ["Video & animation", "mp4 webm mov avi m4v swf"], + ["Archives", "zip 7z rar tar gz bz2 xz"], + ["Documentation & licenses", "txt md pdf rtf doc docx url"], + [ + "3D models", + "stl obj ply fbx dae 3ds gltf glb usdz step stp iges igs skp 3mf amf", + ], +]; +const FAMILIES = new Map( + GROUPS.flatMap(([family, types]) => + types.split(" ").map((ext) => [ext, family]), + ), +); +const REFERENCE_TYPES = new Set( + "json xml tmx tsx tmj tsj world atlas fnt gltf obj mtl dae fbx html css js ts spriteatlas meta mat asset anim controller".split( + " ", + ), +); +const CATEGORIES = [ + ["World & terrain", /world|terrain|tile|dungeon|environment|landscape/], + ["Characters & outfits", /character|outfit|player|hero|npc/], + ["Monsters & creatures", /monster|creature|enemy|enemies/], + ["Items & equipment", /item|equipment|icon|weapon|inventory/], + ["Interface & HUD", /(?:\bui\b|hud|input|interface|button|menu)/], + ["Effects & particles", /vfx|particle|effect/], + ["Audio & music", /audio|music|sound|sfx/], + ["Fonts & text", /font|text/], + ["Maps & levels", /map|level/], + ["Source projects & tools", /source|project|tool/], + ["Documentation & licenses", /documentation|license|catalog/], +]; +function gameDetails(name, relative) { + const ext = path.extname(name).slice(1).toLowerCase(); + const family = + /^(?:F\d+__)?(?:licen[cs]e|copying|copyright|readme)(?:[._-]|$)/i.test(name) + ? "Documentation & licenses" + : FAMILIES.get(ext) || (ext === "mtl" ? "3D materials" : "Other files"); + const folder = + path + .dirname(relative) + .split(/[\\/]/) + .find((x) => x !== "." && x !== "..") || ""; + const normalized = folder + .toLowerCase() + .replace(/[_-]/g, " ") + .replace(/^\d+\s*/, ""); + const match = CATEGORIES.find(([, re]) => re.test(normalized)); + return { + family, + category: match?.[0] || family, + evidence: match + ? `Suggested from source folder “${folder}”. File content and purpose require review.` + : `Grouped by file extension or documentation filename; ${family === "Other files" ? "unknown files are retained for review" : "runtime compatibility is not verified"}.`, + confidence: match ? "Low — source folder" : "Format grouping", + }; +} +module.exports = { gameDetails, REFERENCE_TYPES }; diff --git a/app/game-library.cjs b/app/game-library.cjs new file mode 100644 index 0000000..2f33276 --- /dev/null +++ b/app/game-library.cjs @@ -0,0 +1,510 @@ +const fs = require("node:fs"); +const fsp = require("node:fs/promises"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const readline = require("node:readline"); +const { + hashFile, + unchanged, + readRow, + inside, + rejectLinks, +} = require("./files.cjs"); +const { REFERENCE_TYPES } = require("./game-formats.cjs"); +const slash = (value) => value.replaceAll("\\", "/"); +const safe = (value) => { + const name = + String(value || "Other") + .replace(/[<>:"/\\|?*\x00-\x1f]/g, "_") + .slice(0, 55) + .replace(/[. ]+$/, "") || "Other"; + return /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(name) + ? "_" + name + : name; +}; +const relativeSafe = (value) => + typeof value === "string" && + value.length < 4096 && + !path.win32.isAbsolute(value) && + !path.posix.isAbsolute(value) && + !value.split(/[\\/]/).includes(".."); +async function historicalPaths(roots, signal) { + const records = new Map(), + warnings = []; + for (const root of roots) { + if (signal.aborted) break; + const file = path.join(root, "00_CATALOG", "FILE_INDEX.jsonl"); + try { + await rejectLinks(path.dirname(file)); + const stat = await fsp.lstat(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.size > 200 * 1024 ** 2 + ) + throw new Error( + "Historical index is not a regular file within the 200 MiB limit.", + ); + const stream = fs.createReadStream(file, { signal }); + const lines = readline.createInterface({ + input: stream, + crlfDelay: Infinity, + }); + try { + for await (const line of lines) { + if (signal.aborted) break; + if (line.length > 32768) + throw new Error("Historical index line exceeds limit."); + const value = JSON.parse(line); + if ( + !relativeSafe(value.destination_relative_path) || + !relativeSafe(value.source_relative_path) + ) + throw new Error("Historical index contains unsafe relative paths."); + const current = path.resolve(root, value.destination_relative_path); + const key = current.toLowerCase(); + if (!inside(root, current) || records.has(key)) + throw new Error("Historical index contains ambiguous paths."); + records.set(key, { + sourceRelativePath: slash(value.source_relative_path), + packName: String(value.pack_name || "").slice(0, 300), + packId: String(value.pack_id || "").slice(0, 100), + indexPath: file, + status: "Historical path mapping; stored hashes are not trusted", + }); + } + } finally { + lines.close(); + stream.destroy(); + } + await unchanged({ + path: file, + size: stat.size, + mtime: stat.mtimeMs, + ctime: stat.ctimeMs, + }); + } catch (e) { + if (e.code !== "ENOENT" && !signal.aborted) + warnings.push({ path: file, message: e.message }); + } + } + return { records, warnings }; +} +async function rowHash(row, signal) { + await rejectLinks(path.dirname(row.path)); + await unchanged(row); + const hash = row.member + ? crypto + .createHash("sha256") + .update(await readRow(row)) + .digest("hex") + : await hashFile(row.path, signal); + await unchanged(row); + return hash; +} +async function prepareLibrary(catalog, options, parent, signal, emit) { + catalog.flush(); + const destination = await fsp.realpath(parent); + if (!(await fsp.stat(destination)).isDirectory()) + throw new Error("Choose a destination folder."); + await rejectLinks(parent); + const scope = options.kind === "unique" ? { ...options, kind: "" } : options; + const ids = catalog.ids(scope); + if (!ids.length) throw new Error("No files match this catalog scope."); + const roots = [...new Set(ids.map((id) => catalog.get(id).root))]; + const scannedRoots = [ + ...new Set([ + ...(catalog.setting("lastScan")?.roots || []), + ...catalog.db + .prepare("SELECT DISTINCT root FROM files") + .all() + .map((row) => row.root), + ]), + ]; + if (scannedRoots.some((root) => inside(root, destination))) + throw new Error("Choose a destination outside the scanned folders."); + const id = crypto.randomUUID(); + const directory = path.join( + destination, + `Asset-Library-${new Date().toISOString().replace(/[:.]/g, "-")}-${id.slice(0, 8)}`, + ); + await fsp.mkdir(directory); + const indexDir = path.join(directory, "indexes"); + await fsp.mkdir(indexDir); + const handles = new Map(); + const written = new Set(); + const output = async (filename, data) => { + let handle = handles.get(filename); + if (!handle) { + if (handles.size >= 24) { + const [oldName, oldHandle] = handles.entries().next().value; + await oldHandle.sync(); + await oldHandle.close(); + handles.delete(oldName); + } + handle = await fsp.open( + path.join(directory, filename), + written.has(filename) ? "a" : "wx", + ); + written.add(filename); + handles.set(filename, handle); + } + await handle.write(JSON.stringify(data) + "\n"); + }; + const summary = { + schemaVersion: 1, + id, + created: new Date().toISOString(), + status: "preparing", + roots, + filters: options, + effectiveFilters: scope, + files: ids.length, + processed: 0, + uniqueFiles: 0, + duplicateCopies: 0, + sourceBytes: 0, + uniqueBytes: 0, + redundantBytes: 0, + referenceReview: 0, + errorCount: 0, + errors: [], + historicalWarnings: [], + categories: [], + archivesIncluded: 0, + }; + for (const filename of ["assets.jsonl", "sources.jsonl", "copy-plan.jsonl"]) { + handles.set(filename, await fsp.open(path.join(directory, filename), "wx")); + written.add(filename); + } + const state = () => ({ + phase: "assetCatalog", + running: true, + total: ids.length, + done: summary.processed, + errorCount: summary.errorCount, + errors: summary.errors, + }); + emit(state()); + const { records: history, warnings } = await historicalPaths(roots, signal); + summary.historicalWarnings = warnings; + const unique = new Map(), + categories = new Map(), + categoryMembers = new Set(), + preview = []; + let last = Date.now(); + const hashUpdates = []; + const provenance = catalog.db.prepare( + "SELECT c.name,c.creator,c.license,c.url FROM collections c JOIN collection_members m ON m.collection_id=c.id WHERE m.file_id=?", + ); + try { + for (let start = 0; start < ids.length && !signal.aborted; start += 4) { + const batch = ids.slice(start, start + 4).map((id) => catalog.get(id)); + const hashes = await Promise.allSettled( + batch.map((row) => rowHash(row, signal)), + ); + for (let i = 0; i < batch.length; i++) { + const row = batch[i], + result = hashes[i]; + if (signal.aborted) break; + summary.processed++; + summary.sourceBytes += row.size; + if (result.status === "rejected") { + hashUpdates.push({ id: row.id, hash: null }); + const error = { + path: row.path, + member: row.member, + message: result.reason.message, + }; + summary.errorCount++; + if (summary.errors.length < 30) summary.errors.push(error); + await output("errors.jsonl", error); + continue; + } + const hash = result.value; + if (row.hash !== hash) hashUpdates.push({ id: row.id, hash }); + const category = row.reviewedCategory || row.category, + family = row.family; + let canonical = unique.get(hash); + const historical = history.get(row.path.toLowerCase()); + const source = { + id: row.id, + name: row.name, + extension: row.ext, + path: row.path, + root: row.root, + relativePath: slash(path.relative(row.root, row.path)), + member: row.member, + size: row.size, + modifiedUtc: new Date(row.mtime).toISOString(), + sha256: hash, + category, + family, + tags: row.tags, + notes: row.notes || "", + historical: historical || null, + referenceReview: REFERENCE_TYPES.has(row.ext), + collections: provenance.all(row.id), + }; + if (!canonical) { + const stem = safe(path.parse(row.name).name).slice(0, 36); + const ext = /^[a-z0-9_-]{1,16}$/i.test(row.ext) ? "." + row.ext : ""; + const relative = slash( + path.join( + "Assets", + safe(category), + ...(category === family ? [] : [safe(family)]), + hash.slice(0, 2), + `${stem}__${hash}${ext}`, + ), + ); + canonical = { relative, id: row.id }; + unique.set(hash, canonical); + const asset = { + ...source, + canonicalRelativePath: relative, + copyStatus: "Planned; consult copy-receipt.json before use", + }; + await output("assets.jsonl", asset); + await output("copy-plan.jsonl", { + ...row, + expectedHash: hash, + target: path.join(directory, relative), + }); + summary.uniqueFiles++; + summary.uniqueBytes += row.size; + if (preview.length < 12) + preview.push({ + name: row.name, + category, + family, + relativePath: relative, + size: row.size, + }); + } else { + summary.duplicateCopies++; + summary.redundantBytes += row.size; + } + const membership = category + "\0" + hash; + if (!categoryMembers.has(membership)) { + categoryMembers.add(membership); + const slug = + safe(category) + + "__" + + crypto + .createHash("sha256") + .update(category) + .digest("hex") + .slice(0, 8); + const previous = categories.get(category); + const chunk = slash( + path.join( + "indexes", + slug + + "-" + + String( + Math.floor((previous?.uniqueFiles || 0) / 1000) + 1, + ).padStart(4, "0") + + ".jsonl", + ), + ); + await output(chunk, { + ...source, + canonicalRelativePath: canonical.relative, + }); + categories.set(category, { + category, + index: previous?.index || chunk, + indexes: [...new Set([...(previous?.indexes || []), chunk])], + uniqueFiles: (categories.get(category)?.uniqueFiles || 0) + 1, + }); + } + if (source.referenceReview) summary.referenceReview++; + if (row.member) summary.archivesIncluded++; + await output("sources.jsonl", { + ...source, + canonicalRelativePath: canonical.relative, + canonicalId: canonical.id, + duplicate: canonical.id !== row.id, + }); + } + if (hashUpdates.length >= 256) catalog.setHashes(hashUpdates.splice(0)); + if (Date.now() - last > 250) { + last = Date.now(); + emit(state()); + } + } + catalog.setHashes(hashUpdates); + summary.status = signal.aborted + ? "cancelled" + : summary.errorCount + ? "incomplete" + : "ready"; + summary.categories = [...categories.values()].sort((a, b) => + a.category.localeCompare(b.category), + ); + await fsp.writeFile( + path.join(directory, "summary.json"), + JSON.stringify(summary, null, 2), + { flag: "wx" }, + ); + const markdown = `# Asset library\n\nStatus: **${summary.status}**. ${summary.uniqueFiles.toLocaleString()} unique files; ${summary.duplicateCopies.toLocaleString()} extra copies indexed. Assets are **not copied yet**.\n\n## Read this first\n\n- Open summary.json for scope, counts, errors and provenance warnings. Only status ready permits a reviewed copy.\n- assets.jsonl lists one canonical asset per freshly verified SHA-256. Category indexes below are split into at most 1,000 entries each; use summary.json to locate every chunk. The same hash may appear in several category indexes but has one physical copy.\n- sources.jsonl retains every selected source path and duplicate alias, plus historical original paths where available. Historical hashes are never used for duplicate decisions.\n- copy-plan.jsonl maps verified source fingerprints to the proposed new organization under Assets/category/file family/hash prefix, combining identical category/family levels. Original filenames are shortened and given a SHA-256 suffix; files are not overwritten.\n- Consult copy-receipt.json and copy-journal.jsonl for actual copy status. Missing receipt means no copy has completed.\n- ${summary.referenceReview.toLocaleString()} source files require reference review. Paths and names change in this unique-file library. Maps, atlases, fonts and projects may require reference repair before runtime use. Their source aliases and historical paths are retained; this catalog does not rewrite them.\n- Extension and source-folder categories are suggestions, not proof of identity or Phaser compatibility. Source/editor files may require export or conversion.\n- Preserve license documents and collection provenance. No license is inferred from a filename or category. Use sources.jsonl to find each pack's documents and original paths.\n- Assets remain local; no files or catalog data are sent to a cloud service.\n\n## Category indexes\n\n${summary.categories.map((c) => `- ${c.category.replace(/[\r\n]/g, " ")}: ${c.uniqueFiles.toLocaleString()} unique assets — ${c.indexes.join(", ")}`).join("\n")}\n\n## Using this catalog in another chat\n\nGive the chat the absolute path to this START_HERE.md, then ask it to read summary.json and the relevant category index. Search sources.jsonl for original pack names, names or paths. Resolve canonicalRelativePath against this library directory; until a successful copy, use each record's path/member to locate the source. Archive members require extraction, and archive scans are opt-in. Do not assume a planned target exists.\n`; + await fsp.writeFile(path.join(directory, "START_HERE.md"), markdown, { + flag: "wx", + }); + emit({ + ...state(), + phase: summary.status === "ready" ? "complete" : summary.status, + running: false, + }); + return { + id, + directory, + summary, + preview, + created: Date.now(), + planHash: await hashFile(path.join(directory, "copy-plan.jsonl")), + }; + } finally { + for (const handle of handles.values()) { + await handle.sync(); + await handle.close(); + } + } +} +async function copyLibrary(prepared, signal, emit) { + if (prepared.summary.status !== "ready") + throw new Error("Prepare a complete catalog before reviewing a copy."); + if (prepared.used) + throw new Error("This plan has already been used. Review a new catalog."); + if (Date.now() - prepared.created > 24 * 60 * 60 * 1000) + throw new Error("Plan expired. Prepare and review a new catalog."); + prepared.used = true; + const directory = prepared.directory; + await rejectLinks(directory); + if ( + (await hashFile(path.join(directory, "copy-plan.jsonl"))) !== + prepared.planHash + ) + throw new Error("Copy plan changed. Prepare and review a new catalog."); + const journal = await fsp.open( + path.join(directory, "copy-journal.jsonl"), + "ax", + ); + const receipt = { + id: prepared.id, + started: new Date().toISOString(), + status: "copying", + total: prepared.summary.uniqueFiles, + done: 0, + copied: 0, + failed: 0, + errors: [], + sourceFilesModified: false, + }; + let stream, lines; + const record = async (data) => { + await journal.write( + JSON.stringify({ at: new Date().toISOString(), ...data }) + "\n", + ); + await journal.sync(); + }; + try { + await record({ event: "start", id: prepared.id }); + stream = fs.createReadStream(path.join(directory, "copy-plan.jsonl")); + lines = readline.createInterface({ input: stream, crlfDelay: Infinity }); + for await (const line of lines) { + if (signal.aborted) break; + const row = JSON.parse(line); + let created = false; + try { + if ( + !inside(path.join(directory, "Assets"), row.target) || + !/^[a-f0-9]{64}$/.test(row.expectedHash) + ) + throw new Error("Invalid copy plan entry."); + await rejectLinks(path.dirname(row.target)); + await unchanged(row); + const before = await rowHash(row, signal); + if (before !== row.expectedHash) + throw new Error("Source hash changed since review. Source retained."); + await fsp.mkdir(path.dirname(row.target), { recursive: true }); + await rejectLinks(path.dirname(row.target)); + await record({ + event: "intent", + source: row.path, + member: row.member, + target: row.target, + }); + if (row.member) + await fsp.writeFile(row.target, await readRow(row), { flag: "wx" }); + else + await fsp.copyFile(row.path, row.target, fs.constants.COPYFILE_EXCL); + created = true; + const target = await fsp.open(row.target, "r+"); + try { + await target.sync(); + } finally { + await target.close(); + } + if ((await hashFile(row.target)) !== row.expectedHash) + throw new Error("Copy hash verification failed. Source retained."); + await unchanged(row); + if ((await rowHash(row, signal)) !== row.expectedHash) + throw new Error("Source changed during copying. Source retained."); + await record({ + event: "verified", + source: row.path, + target: row.target, + sha256: row.expectedHash, + }); + receipt.copied++; + } catch (e) { + const error = { + source: row.path, + target: row.target, + message: e.message, + created, + }; + receipt.failed++; + if (receipt.errors.length < 30) receipt.errors.push(error); + await record({ event: "failed", ...error }); + } + receipt.done++; + emit({ + phase: "assetCopy", + running: true, + done: receipt.done, + total: receipt.total, + errorCount: receipt.failed, + errors: receipt.errors, + }); + } + receipt.status = signal.aborted + ? "cancelled" + : receipt.failed + ? "incomplete" + : "complete"; + receipt.finished = new Date().toISOString(); + await record({ event: "finished", ...receipt }); + await fsp.writeFile( + path.join(directory, "copy-receipt.json"), + JSON.stringify(receipt, null, 2), + { flag: "wx" }, + ); + emit({ + phase: receipt.status, + running: false, + done: receipt.done, + total: receipt.total, + errorCount: receipt.failed, + errors: receipt.errors, + }); + return receipt; + } finally { + lines?.close(); + stream?.destroy(); + await journal.close(); + } +} +module.exports = { prepareLibrary, copyLibrary }; diff --git a/app/main.cjs b/app/main.cjs index 045b264..735d1f8 100644 --- a/app/main.cjs +++ b/app/main.cjs @@ -42,6 +42,7 @@ let win, jobState = null, plan = null, transferring = false, + assetPlan = null, apiKey = ""; const progress = (data) => { jobState = data; @@ -81,8 +82,31 @@ else app.whenReady().then(async () => { await fsp.mkdir(app.getPath("userData"), { recursive: true }); catalog = new Catalog(path.join(app.getPath("userData"), "catalog.sqlite")); - protocol.handle("scout", (request) => { + protocol.handle("scout", async (request) => { const url = new URL(request.url); + if ( + url.hostname === "app" && + /^\/image\/[a-f0-9]{64}$/.test(url.pathname) + ) { + const image = catalog.get(url.pathname.split("/").pop()); + const mime = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + webp: "image/webp", + gif: "image/gif", + bmp: "image/bmp", + }[image?.ext]; + if (!mime || image.size > 8 * 1024 ** 2) + return new Response("Image preview unavailable", { status: 413 }); + try { + return new Response(await readRow(image), { + headers: { "Content-Type": mime, "Cache-Control": "no-store" }, + }); + } catch { + return new Response("Source changed or unavailable", { status: 409 }); + } + } if ( url.hostname === "app" && /^\/thumbnail\/[a-f0-9]{64}$/.test(url.pathname) @@ -97,7 +121,7 @@ else }) : new Response("Not found", { status: 404 }); } - const root = path.join(__dirname, "..", "dist"); + const root = path.join(__dirname, "..", "dist", app.getVersion()); const target = path.resolve( root, "." + @@ -183,6 +207,7 @@ else const previousScan = catalog.setting("lastScan"); thumbnails.stop(); plan = null; + assetPlan = null; scan(options, catalog, job.signal, progress) .catch((e) => { catalog.finishScan(true); @@ -409,6 +434,62 @@ else handle("reveal", (id) => { shell.showItemInFolder(row(id).path); }); + handle("assetPrepare", async (options, destination) => { + requireIdle(); + job = new AbortController(); + thumbnails.stop(); + assetPlan = null; + try { + assetPlan = await require("./game-library.cjs").prepareLibrary( + catalog, + options || {}, + destination, + job.signal, + progress, + ); + return { + id: assetPlan.id, + directory: assetPlan.directory, + summary: assetPlan.summary, + preview: assetPlan.preview, + }; + } catch (e) { + progress({ + phase: "failed", + running: false, + message: e.message, + errorCount: 1, + errors: [{ path: destination, message: e.message }], + }); + throw e; + } finally { + job = null; + } + }); + handle("assetCopy", async (id) => { + requireIdle(); + if (!assetPlan || assetPlan.id !== id) + throw new Error("Prepare and review an asset catalog first."); + job = new AbortController(); + thumbnails.stop(); + try { + return await require("./game-library.cjs").copyLibrary( + assetPlan, + job.signal, + progress, + ); + } catch (e) { + progress({ phase: "failed", running: false, message: e.message }); + throw e; + } finally { + job = null; + } + }); + handle("revealAssetCatalog", (id) => { + if (!assetPlan || assetPlan.id !== id) + throw new Error("Prepare an asset catalog first."); + shell.showItemInFolder(path.join(assetPlan.directory, "START_HERE.md")); + }); handle("exportCsv", async (options) => { const { filePath, canceled } = await dialog.showSaveDialog(win, { title: "Export filtered results", diff --git a/app/preload.cjs b/app/preload.cjs index 6aa0b63..332db7b 100644 --- a/app/preload.cjs +++ b/app/preload.cjs @@ -25,6 +25,9 @@ const names = [ "transferExecute", "reveal", "exportCsv", + "assetPrepare", + "assetCopy", + "revealAssetCatalog", "ai", "setKey", "metadata", diff --git a/app/scanner.cjs b/app/scanner.cjs index 11b1808..73a2f2c 100644 --- a/app/scanner.cjs +++ b/app/scanner.cjs @@ -3,11 +3,12 @@ const path = require("node:path"); const { EXTENSIONS, EXTRA_EXTENSIONS, fileRecord } = require("./formats.cjs"); const { listArchive } = require("./archive.cjs"); async function scan( - { roots, archives = false, extras = false }, + { roots, archives = false, extras = false, mode = "models" }, catalog, signal, onProgress, ) { + if (!["models", "game"].includes(mode)) throw new Error("Unknown scan mode."); const extensions = new Set([ ...EXTENSIONS, ...(extras ? EXTRA_EXTENSIONS : []), @@ -34,6 +35,7 @@ async function scan( catalog.beginScan(); const state = { phase: "files", + mode, directories: 0, files: 0, matches: 0, @@ -81,16 +83,17 @@ async function scan( if (!entry.isFile()) continue; state.files++; const ext = path.extname(entry.name).slice(1).toLowerCase(); - if (extensions.has(ext)) { + if (["zip", "7z", "rar"].includes(ext)) { + catalog.addLocation(file, "archive", root); + if (archives) packs.push({ file, root }); + } + if (mode === "game" || extensions.has(ext)) { try { - catalog.add(fileRecord(file, await fs.stat(file), root)); + catalog.add(fileRecord(file, await fs.stat(file), root, "", mode)); state.matches++; } catch (e) { error(file, e); } - } else if (["zip", "7z", "rar"].includes(ext)) { - catalog.addLocation(file, "archive", root); - if (archives) packs.push({ file, root }); } emit(); } @@ -115,6 +118,7 @@ async function scan( file, async (entry) => { if ( + mode === "game" || extensions.has(path.extname(entry.name).slice(1).toLowerCase()) ) { catalog.add( @@ -127,6 +131,7 @@ async function scan( }, root, entry.name, + mode, ), ); state.matches++; @@ -143,6 +148,7 @@ async function scan( } } catalog.finishScan(signal.aborted); + if (signal.aborted) Object.assign(state, catalog.setting("lastScan")); state.phase = signal.aborted ? "cancelled" : "complete"; state.running = false; state.restoredPrevious = signal.aborted; diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5a1bb0b..184a110 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -7,6 +7,20 @@ GitHub had no open issues or pull requests at assessment time. That does not mea the product backlog is empty. The items below distinguish reproduced defects from proposed features. The reliability fixes below are implemented in 0.4.1. +## 0.5.2 game asset library + +Implemented: all-regular-file game scanning, family/extensionless filters, local +raster previews, unique results, and reviewed creation of a new category/family +library with one freshly verified copy per SHA-256. Markdown/JSONL catalogs keep +source aliases, optional historical pack paths, provenance and bounded category +indexes. Source files remain untouched. Preparation expands unique-result +collapsing to retain aliases, checks all scanned roots, and scan recovery retains +the previous catalog mode and locations. + +Reference rewriting and source/editor conversion remain future work. The current +library is a source inventory and verified copy, with reference review flags; +it does not claim ready-to-load Phaser assets or continuous synchronization. + ## 0.4.1 resolution Filename search is now the default. Direct folder and archive name matches appear diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index a919bbf..4bc692e 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -4,6 +4,8 @@ Use Windows x64 and Node.js 24.15 or newer. Tests create disposable files under .test-data and isolated app profiles. They never use a live user catalog. +Set `$env:SCOUT_RETAIN_FIXTURES='1'` before these commands unless fixture removal +has been explicitly authorized. Keep generated outputs and release candidates. 1. Run npm ci and npm test for scanning, archives, duplicate hashes, exclusive verified transfers, cancellation recovery, metadata, collections, caches, @@ -19,7 +21,59 @@ Use Windows x64 and Node.js 24.15 or newer. Tests create disposable files under WASM hashes against pinned upstream revisions and packages sources/notices with the executable. Keep the generated SHA-256 file with the ZIP. -## Local acceptance, 0.4.1 +## Local acceptance, 0.5.2 + +Use `SCOUT_RETAIN_FIXTURES=1` to retain generated originals and collections. +This mode passes 31 core/feature tests and all eight source desktop workflows, +plus the same eight workflows against the unpacked Windows release executable. +It skips four core move/relocation checks, substitutes copying for the duplicate +move UI check, and omits collection-removal UI execution. CI uses this mode. +Do not describe it as complete move/delete coverage. The full 31-test source +suite also passed locally in this session, but an attempted anchored negative +name filter failed to exclude its destructive fixture checks. The behavior was +reproduced in a harmless two-test fixture; positive `--test-skip-pattern` exclusions +were verified before enabling retained-fixture checks. That earlier 31-test run +preceded the category chunking regression added in 0.5.2. No personal libraries or +live app profiles were used by tests. + +New coverage: model mode unchanged; all-file game discovery, unknown/extensionless +types, opt-in archive members, family/saved filters, scoped unique results, +fresh SHA-256 deduplication, aliases/historical paths, category-index writer +rotation, reviewed copy, source retention, changed/tampered-plan rejection, +1,000-entry category chunks and hash-prefix asset folders, +destination containment, cancellation, raster previews/gallery, persisted scan +mode and minimum supported window size. Copy tests verify actual file bytes, +source fingerprints, catalogs and receipts. Native dialogs/reveal and cloud calls +remain intercepted or mocked where applicable. + +Independent review identified three additional regressions: unique-view +preparation omitted aliases, filtered destination checks omitted other scan +roots, and cancelled mode changes mislabeled the restored catalog. All three +failed newly added tests before fixes and now pass. Coverage includes an empty +scan root, stale duplicate aliases, process-reopen mode recovery, and desktop +preparation from unique results. + +Two earlier hosted runs intermittently timed out on the cancelled-mode family +selector check while paired runs on identical commits passed. Ten repeated local +game workflows and a 6x renderer-throttled run passed. Failure diagnostics were +retained; both hosted Windows runs on diagnostic head `c2d0db6` passed. No +production race or root cause was confirmed, so this is a known verification +uncertainty rather than a claimed application fix. + +A read-only large-library scan found 201,335 files across 609 directories, with +zero errors, in 25.09 seconds; average unfiltered gallery-page query was 19.42 ms. +A separate complete SHA-256 pass found 160,096 unique hashes and 41,239 extra +copies, with zero read/fingerprint errors. Timing includes storage/cache effects. +This does not establish the runtime of a full-library prepared export or copy; +development did not copy the owner's library. Receipts and paths remain private +under ignored artifacts. + +Format checks, build, dependency audit and manifest-based portable packaging +passed. Packaging retains earlier generated outputs and includes only current +manifest files. The build is unsigned. Maps/atlases/projects are flagged for +reference review; reference rewriting and source conversion are not implemented. + +## Historical local acceptance, 0.4.1 The Windows source suite has 27 passing core/feature tests. Seven source desktop workflows and the same seven packaged workflows pass. Desktop fixtures cover diff --git a/package-lock.json b/package-lock.json index b8616d5..e75784e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "model-scout", - "version": "0.4.1", + "version": "0.5.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "model-scout", - "version": "0.4.1", + "version": "0.5.2", "license": "MIT", "dependencies": { "assimpjs": "^0.0.10", diff --git a/package.json b/package.json index b8ae559..0050f5a 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,17 @@ { "name": "model-scout", - "version": "0.4.1", + "version": "0.5.2", "private": true, - "description": "A local 3D file discovery and organization workbench", + "description": "A local model and game asset discovery and organization workbench", "author": "Big Bwain LLC", "license": "MIT", "main": "app/main.cjs", "scripts": { "build": "vite build", "start": "npm run build && electron .", - "test": "node --test tests/core.test.cjs tests/enhancements.test.cjs tests/organization.test.cjs tests/geometry.test.cjs tests/reliability.test.cjs", - "test:app": "node tests/app.test.cjs && node tests/library-app.test.cjs && node tests/production-app.test.cjs && node tests/organization-app.test.cjs && node tests/importers-app.test.cjs && node tests/geometry-app.test.cjs && node tests/reliability-app.test.cjs", - "package": "npm run licenses && npm run build && electron-builder --win portable --x64", + "test": "node scripts/test-core.cjs", + "test:app": "node tests/app.test.cjs && node tests/library-app.test.cjs && node tests/production-app.test.cjs && node tests/organization-app.test.cjs && node tests/importers-app.test.cjs && node tests/geometry-app.test.cjs && node tests/reliability-app.test.cjs && node tests/game-assets-app.test.cjs", + "package": "npm run licenses && npm run build && node scripts/package.cjs", "format:check": "prettier --check app src tests scripts vite.config.mjs package.json", "test:packaged": "node scripts/test-packaged.cjs", "licenses": "node scripts/licenses.cjs", @@ -21,12 +21,12 @@ "appId": "com.bigbwain.modelscout", "productName": "Model Scout", "directories": { - "output": "release/0.4.1" + "output": "release/0.5.2" }, "files": [ "app/**/*", "src/costing.mjs", - "dist/**/*", + "dist/0.5.2/**/*", "package.json", "docs/THIRD_PARTY.md", "notices/**/*", diff --git a/scripts/benchmark.cjs b/scripts/benchmark.cjs index 630580b..cb69de7 100644 --- a/scripts/benchmark.cjs +++ b/scripts/benchmark.cjs @@ -6,7 +6,7 @@ const { duplicates } = require("../app/files.cjs"); (async () => { if (!process.argv[2] || process.argv[2].startsWith("--")) throw new Error( - "Usage: node scripts/benchmark.cjs [--duplicates]", + "Usage: node scripts/benchmark.cjs [--game] [--duplicates]", ); const root = path.resolve(process.argv[2]); const output = path.resolve("artifacts", "benchmarks", String(Date.now())); @@ -21,7 +21,11 @@ const { duplicates } = require("../app/files.cjs"); try { let start = performance.now(); receipt.scan = await scan( - { roots: [root], archives: false }, + { + roots: [root], + archives: false, + mode: process.argv.includes("--game") ? "game" : "models", + }, c, signal, () => {}, diff --git a/scripts/package.cjs b/scripts/package.cjs new file mode 100644 index 0000000..23a0541 --- /dev/null +++ b/scripts/package.cjs @@ -0,0 +1,50 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { execFileSync } = require("node:child_process"); +const { version, build } = require("../package.json"); +const root = path.resolve("dist", version); +const manifest = JSON.parse( + fs.readFileSync(path.join(root, ".vite", "manifest.json"), "utf8"), +); +const outputs = new Set(["index.html", "thumbnail.html"]); +for (const entry of Object.values(manifest)) + for (const file of [ + entry.file, + ...(entry.css || []), + ...(entry.assets || []), + ]) { + if (path.isAbsolute(file) || file.split(/[\\/]/).includes("..")) + throw new Error("Build manifest contains an unsafe path."); + outputs.add(file); + } +const config = { + ...build, + files: [ + ...build.files.filter( + (file) => typeof file !== "string" || !file.startsWith("dist/"), + ), + ...[...outputs].map((file) => `dist/${version}/${file}`), + ], +}; +if (fs.existsSync(path.join(build.directories.output, "win-unpacked"))) + throw new Error( + "Unpacked release already exists. Choose a fresh build output directory to retain the previous build.", + ); +fs.mkdirSync(".local", { recursive: true }); +const configPath = path.resolve( + ".local", + `package-${version}-${Date.now()}.json`, +); +fs.writeFileSync(configPath, JSON.stringify(config, null, 2), { flag: "wx" }); +execFileSync( + process.execPath, + [ + require.resolve("electron-builder/cli.js"), + "--win", + "portable", + "--x64", + "--config", + configPath, + ], + { stdio: "inherit" }, +); diff --git a/scripts/release-bundle.cjs b/scripts/release-bundle.cjs index 950a4e9..024127e 100644 --- a/scripts/release-bundle.cjs +++ b/scripts/release-bundle.cjs @@ -113,6 +113,18 @@ async function download(url) { `${hash} ${executable}\n`, ); const zip = path.join(out, `Model-Scout-${version}-Windows-x64.zip`); + if ( + await fs.stat(zip).then( + () => true, + (e) => { + if (e.code === "ENOENT") return false; + throw e; + }, + ) + ) + throw new Error( + "Release ZIP already exists. Retain it and choose a fresh output before bundling again.", + ); const quote = (value) => "'" + value.replaceAll("'", "''") + "'"; const entries = (await fs.readdir(content)) .map((file) => quote(path.join(content, file))) @@ -123,7 +135,7 @@ async function download(url) { "-NoProfile", "-NonInteractive", "-Command", - `Compress-Archive -LiteralPath @(${entries}) -DestinationPath ${quote(zip)} -Force`, + `Compress-Archive -LiteralPath @(${entries}) -DestinationPath ${quote(zip)}`, ], { stdio: "inherit", windowsHide: true }, ); diff --git a/scripts/test-core.cjs b/scripts/test-core.cjs new file mode 100644 index 0000000..5a8581e --- /dev/null +++ b/scripts/test-core.cjs @@ -0,0 +1,22 @@ +const { spawnSync } = require("node:child_process"); +const args = ["--test"]; +if (process.env.SCOUT_RETAIN_FIXTURES === "1") + args.push( + "--test-skip-pattern=copy preserves source|collections persist|OBJ transfers|cached comparison resumes", + ); +args.push( + ...[ + "core", + "enhancements", + "organization", + "geometry", + "reliability", + "game-assets", + ].map((name) => `tests/${name}.test.cjs`), +); +const result = spawnSync(process.execPath, args, { + stdio: "inherit", + env: process.env, +}); +if (result.error) throw result.error; +process.exitCode = result.status || 0; diff --git a/scripts/test-packaged.cjs b/scripts/test-packaged.cjs index ab3a84d..d9612d8 100644 --- a/scripts/test-packaged.cjs +++ b/scripts/test-packaged.cjs @@ -7,6 +7,7 @@ for (const file of [ "importers-app", "geometry-app", "reliability-app", + "game-assets-app", ]) { const result = spawnSync(process.execPath, [`tests/${file}.test.cjs`], { stdio: "inherit", diff --git a/src/AssetLibrary.jsx b/src/AssetLibrary.jsx new file mode 100644 index 0000000..16e74c7 --- /dev/null +++ b/src/AssetLibrary.jsx @@ -0,0 +1,238 @@ +import React, { useState } from "react"; +const size = (n) => { + if (!n) return "0 B"; + const k = Math.min(4, Math.floor(Math.log(n) / Math.log(1024))); + return `${(n / 1024 ** k).toFixed(k ? 1 : 0)} ${["B", "KiB", "MiB", "GiB", "TiB"][k]}`; +}; +export function AssetLibrary({ query, running, progress, onError }) { + const [destination, setDestination] = useState(""); + const [scope, setScope] = useState("all"); + const [prepared, setPrepared] = useState(null); + const [receipt, setReceipt] = useState(null); + const [busy, setBusy] = useState(false); + const [reviewed, setReviewed] = useState(false); + const [attempted, setAttempted] = useState(false); + const action = async (fn) => { + setBusy(true); + try { + await fn(); + } catch (e) { + onError(e.message); + } finally { + setBusy(false); + } + }; + const locked = busy || running; + return ( +
+

+ Create a new library containing one copy of each unique file. Original + files stay in place. Categories and file families form the new folder + structure; indexes retain every source path and duplicate alias. +

+ + {scope === "filtered" && query.kind === "unique" && ( +

+ The Unique files view is expanded for preparation so every matching + source alias is freshly verified and retained. Other filters still + apply. +

+ )} + +

+ A new Asset-Library folder will be created here. Preparation reads and + hashes files, then writes the catalog and copy plan. It does not copy + the assets yet. Archive contents are included only if enabled in the + scan. +

+ + {locked && ( +

+ {progress?.phase === "assetCopy" + ? "Copying and verifying" + : "Preparing catalog"} + : {(progress?.done || 0).toLocaleString()} /{" "} + {(progress?.total || 0).toLocaleString()}. Files already written are + retained. +

+ )} + {running && ( + + )} + {prepared && ( +
+

Review the new organization

+

+ + {prepared.summary.uniqueFiles.toLocaleString()} unique files + {" "} + · {size(prepared.summary.uniqueBytes)} to copy +
+ {prepared.summary.duplicateCopies.toLocaleString()} extra copies + excluded · {size(prepared.summary.redundantBytes)} saved +

+

{prepared.directory}

+

+ {prepared.summary.referenceReview.toLocaleString()} source files + flagged for reference review. Maps, atlases, fonts and projects may + need updated references before game use. This library changes paths + and filenames and retains mappings; it does not repair references. +

+ {prepared.summary.historicalWarnings.length > 0 && ( +

+ Historical path mapping warnings:{" "} + {prepared.summary.historicalWarnings + .map((x) => x.message) + .join("; ")} +

+ )} + {prepared.summary.status !== "ready" && ( +

+ Catalog is {prepared.summary.status}.{" "} + {prepared.summary.errorCount} file errors. Resolve issues and + prepare again before copying. +

+ )} +
+ + + + + + + + + {prepared.preview.map((item) => ( + + + + + ))} + +
Sample fileNew folder
{item.name} + {item.category} + {item.category !== item.family ? " / " + item.family : ""} +
+
+

+ START_HERE.md explains the catalog. assets.jsonl lists unique + assets; sources.jsonl retains aliases and pack information; category + indexes allow another chat to read smaller lists. copy-plan.jsonl + contains the full planned file-by-file destinations. License + documents are indexed; licensing is not inferred. +

+ + + + {receipt && ( +

+ Copy {receipt.status}: {receipt.copied.toLocaleString()} verified + copies, {receipt.failed.toLocaleString()} failures. Source files + retained. See copy-receipt.json and copy-journal.jsonl for + details. +

+ )} +
+ )} +
+ ); +} diff --git a/src/Gallery.jsx b/src/Gallery.jsx index 6ebc465..33b6c4f 100644 --- a/src/Gallery.jsx +++ b/src/Gallery.jsx @@ -1,4 +1,5 @@ import React, { useEffect, useRef, useState } from "react"; +import { RasterPreview, hasRasterPreview } from "./RasterPreview"; import { Box, Star, @@ -111,7 +112,11 @@ export function Gallery({ onClick={() => onOpen(r)} >
- {r.hasThumbnail ? ( + {hasRasterPreview(r) && + (visible.includes(r.id) || prepare) && + !paused ? ( + + ) : r.hasThumbnail ? ( {"3D onFilter({ family: "" }) }] + : []), ...(query.category ? [{ label: query.category, remove: () => onFilter({ category: "" }) }] : []), @@ -49,6 +52,7 @@ export function SearchTools({ query, onFilter, tags, onSave }) { ext: "", category: "", kind: "", + family: "", collection: "", tags: [], excludeTags: [], diff --git a/src/RasterPreview.jsx b/src/RasterPreview.jsx new file mode 100644 index 0000000..3a5be87 --- /dev/null +++ b/src/RasterPreview.jsx @@ -0,0 +1,22 @@ +import React, { useState } from "react"; +export const hasRasterPreview = (file) => + ["png", "jpg", "jpeg", "webp", "gif", "bmp"].includes(file?.ext) && + file.size <= 8 * 1024 ** 2; +export function RasterPreview({ file }) { + const [failed, setFailed] = useState(false); + return ( +
+ {failed ? ( +

+ Image preview unavailable. The file is still listed and can be copied. +

+ ) : ( + {"Image setFailed(true)} + /> + )} +
+ ); +} diff --git a/src/main.jsx b/src/main.jsx index e290b39..e27fdcf 100644 --- a/src/main.jsx +++ b/src/main.jsx @@ -39,6 +39,8 @@ import { CostPanel } from "./CostPanel"; import { MeshPanel } from "./MeshPanel"; import { CollectionList, CollectionEditor } from "./Collections"; import { GeometryPanel } from "./GeometryPanel"; +import { AssetLibrary } from "./AssetLibrary"; +import { RasterPreview, hasRasterPreview } from "./RasterPreview"; import "./styles.css"; const api = window.scout; const bytes = (n) => { @@ -55,7 +57,7 @@ function Button({ icon: Icon, children, className = "", ...props }) { ); } -function Modal({ title, children, onClose, wide = false }) { +function Modal({ title, children, onClose, wide = false, canClose = true }) { const ref = useRef(); useEffect(() => { ref.current.showModal(); @@ -64,13 +66,17 @@ function Modal({ title, children, onClose, wide = false }) { { + event.preventDefault(); + if (canClose) onClose(); + }} >

{title}

@@ -498,6 +520,24 @@ function App() {
+ + {scanMode === "game" && ( +

+ Images, audio, maps, fonts, code, editor sources, licenses and + unknown types. No extension exclusions. +

+ )}
+ {catalogMode === "game" && ( + + )}
{stats.extensions.map((e) => ( - ))} @@ -1054,7 +1131,7 @@ function App() { - {r.ext} + {r.ext || "none"} {bytes(r.size)} @@ -1141,12 +1218,18 @@ function App() {