diff --git a/.github/workflows/instant-sync.yml b/.github/workflows/instant-sync.yml deleted file mode 100644 index 4d980e1f..00000000 --- a/.github/workflows/instant-sync.yml +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Instant Forge Sync - Triggers propagation to all forges on push/release -name: Instant Sync - -on: - push: - branches: [main, master] - release: - types: [published] - -permissions: - contents: read - -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - name: Trigger Propagation - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v3 - with: - token: ${{ secrets.FARM_DISPATCH_TOKEN }} - repository: hyperpolymath/.git-private-farm - event-type: propagate - client-payload: |- - { - "repo": "${{ github.event.repository.name }}", - "ref": "${{ github.ref }}", - "sha": "${{ github.sha }}", - "forges": "" - } - - - name: Confirm - env: - REPO_NAME: ${{ github.event.repository.name }} - run: echo "::notice::Propagation triggered for ${REPO_NAME}" - - - name: K9-SVC Validation - run: | - echo "Running K9-SVC contractile validation..." - if [ -f .machine_readable/contractiles/must/Mustfile.a2ml ]; then - echo "✅ Mustfile found - running validation" - # Placeholder for actual K9 validation - echo "K9 validation would run here" - else - echo "❌ Mustfile not found" - exit 1 - fi - - - name: Contractile Check - run: | - echo "Checking contractile completeness..." - contractiles=("must" "trust" "dust" "lust" "adjust" "intend") - missing=0 - for c in "${contractiles[@]}"; do - if [ ! -f ".machine_readable/contractiles/$c/${c^}file.a2ml" ]; then - echo "❌ Missing: $c" - missing=$((missing + 1)) - fi - done - if [ $missing -gt 0 ]; then - echo "❌ $missing contractiles missing" - exit 1 - fi - echo "✅ All contractiles present" diff --git a/.github/workflows/placement-guard.yml b/.github/workflows/placement-guard.yml index ede90a38..31d5aff9 100644 --- a/.github/workflows/placement-guard.yml +++ b/.github/workflows/placement-guard.yml @@ -56,7 +56,15 @@ jobs: ALLOW="$ALLOW"'|^(flake\.nix|guix\.scm|Justfile|contractile\.just|stapeln\.toml|opsm\.toml|setup\.sh|\.gitignore|\.gitattributes|\.editorconfig|\.gitlab-ci\.yml|\.nojekyll)$' # Legacy per-database dirs (grandfathered: warn, do not fail — being extracted). - GRANDFATHER='^(verisimdb|lithoglyph|quandledb|nqc|typeql-experimental|verisim-core|verisim-modular-experiment)/' + # + # `lithoglyph` was removed from this list on 2026-07-27. Its extraction is + # complete (hyperpolymath/lithoglyph#4) and its 819 files are gone from this + # repo, so a new file under lithoglyph/ is no longer "legacy content not yet + # moved" — it is fresh duplication of a repo that already exists. Warning + # would let exactly the defect the extraction fixed grow back. It now fails. + # + # Move a directory out of this list as each extraction completes. + GRANDFATHER='^(verisimdb|quandledb|nqc|typeql-experimental|verisim-core|verisim-modular-experiment)/' FAIL=0 while IFS= read -r f; do diff --git a/CLAUDE.md b/CLAUDE.md index eed665e7..073b0bab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,12 @@ See **`REGISTRY.adoc`** for the authoritative map. Examples: VeriSimDB → ### Transitional note -The directories `verisimdb/`, `lithoglyph/`, `quandledb/`, `nqc/`, +`lithoglyph/` is **done**: extraction completed 2026-07-27 and its 819 files were +removed, leaving a pointer README. Everything it held is preserved at its original +path in the `split-history/lithoglyph` tag on `origin` — never prune that tag or the +`_split_lithoglyph` branch. + +The directories `verisimdb/`, `quandledb/`, `nqc/`, `typeql-experimental/`, `verisim-core/`, and `verisim-modular-experiment/` are **legacy content being extracted** to their own repos — see `docs/migration/RESITE-DATABASES-TO-OWN-REPOS.adoc`. **Do not grow them.** A CI guard diff --git a/REGISTRY.adoc b/REGISTRY.adoc index ce132c6b..3f5dabbd 100644 --- a/REGISTRY.adoc +++ b/REGISTRY.adoc @@ -74,8 +74,19 @@ case) and `docs/LITHOGLYPH.adoc` (what it gives Lithoglyph specifically). == Reference strategy (after extraction) -Hybrid: `verisimdb` and `lithoglyph` are git submodules of this coordination repo; every other -repo is referenced by link from this registry. +Every repo is referenced by link from this registry. There are no git submodules. + +This previously read "`verisimdb` and `lithoglyph` are git submodules of this +coordination repo". That was never true of either: `git ls-files --stage` showed both as +ordinary tracked files, not gitlinks, and no `.gitmodules` has ever existed here. +Corrected in favour of what the repo does rather than adding submodule machinery to +make a stale sentence true — a submodule would have to be fetched with +`submodules: recursive` in every workflow that touches the tree, which is real cost for +a coordination repo that only needs to *point* at its members. + +`lithoglyph/` now holds a pointer README only; its 819 files were removed once the +extraction completed (2026-07-27). `verisimdb/` is still 714 tracked files awaiting the +same treatment. == What belongs in *this* (coordination) repo diff --git a/lithoglyph/.bot_directives b/lithoglyph/.bot_directives deleted file mode 120000 index 28115c09..00000000 --- a/lithoglyph/.bot_directives +++ /dev/null @@ -1 +0,0 @@ -.machine_readable/bot_directives \ No newline at end of file diff --git a/lithoglyph/.claude/CLAUDE.md b/lithoglyph/.claude/CLAUDE.md deleted file mode 100644 index 384ae897..00000000 --- a/lithoglyph/.claude/CLAUDE.md +++ /dev/null @@ -1,173 +0,0 @@ -# CLAUDE.md - AI Assistant Instructions for Lithoglyph - -## Project Overview - -Lithoglyph (formerly "Lith") is a narrative-first, reversible, audit-grade database core. Every mutation is a story event with full provenance. The tagline: "The database where the database is part of the story." - -Multi-language stack in dependency order: -1. **Forth** (core-forth/) — Block storage kernel, journaling, data model -2. **Zig** (core-zig/) — C ABI bridge with WAL, 19 functions, block allocator -3. **Zig** (ffi/zig/) — Delegation layer that forwards to core-zig -4. **Idris2** (src/Lith/) — Dependent-type ABI proofs, memory layout verification -5. **Factor** (core-factor/) — GQL runtime: parser, planner, executor -6. **Lean 4** (normalizer/) — Normalization proofs, FD discovery (52 proofs) -7. **Lean 4** (gql-dt/) — Dependently-typed Glyph Query Language -8. **Zig + Rust** (beam/) — BEAM NIFs for Elixir/Erlang integration -9. **Elixir** (lith-http/) — Phoenix HTTP API, control plane -10. **Rust + Web** (studio/) — Gossamer desktop GUI (mock data, early stage) -11. **Zig** (api/) — REST + gRPC API (BROKEN — needs Zig 0.15.2 migration) - -Subprojects with own `.machine_readable/`: gql-dt/, glyphbase/, lith-http/ - -## Architecture - -``` -svalinn (TLS) → lith-http (Elixir :4000) → BEAM NIFs → core-zig (C ABI) → core-forth (blocks) - ↑ - Idris2 ABI proofs (src/Lith/) verify bridge contracts - Lean 4 normalizer verifies schema correctness - GQL-DT (Lean 4) verifies query correctness - Factor GQL plans and executes queries -``` - -## Critical Invariants - -1. **IP Rename PENDING**: Code uses `Lith`/`Form` internally. Must rename to `Litho`/`Lithoglyph` before any public release. Keep "form" only where it literally means HTML/data forms. -2. **Zero `believe_me`** in Idris2 ABI (`src/Lith/`). This is a HARD invariant — no exceptions. -3. **Zero `sorry`** in Lean 4 proofs (normalizer/, gql-dt/). All proofs must be constructive. -4. **`core-zig/src/bridge.zig`** is the WORKING implementation. `ffi/zig/` only delegates to it. Never add new functionality to `ffi/zig/` directly. -5. **Proven library** lives at `/var/mnt/eclipse/repos/proven/` (104+ modules) — never bundle a copy. -6. All Zig `@ptrCast`/`@alignCast`/`@intToPtr` must have `// SAFETY:` comments. -7. **SCM files** ONLY in `.machine_readable/` — never in root directories. -8. **Container runtime**: Podman, never Docker. Files: `Containerfile`, never `Dockerfile`. -9. **Base images**: `cgr.dev/chainguard/wolfi-base:latest` or `cgr.dev/chainguard/static:latest`. - -## Machine-Readable Artefacts - -`.machine_readable/` contains: -- `STATE.scm` — Current project state, component status, completion percentages -- `META.scm` — Architecture decisions, development practices, design rationale -- `ECOSYSTEM.scm` — Position in ecosystem, related projects -- `AGENTIC.scm` — AI agent interaction patterns, autonomous agent designs -- `NEUROSYM.scm` — Neurosymbolic integration (symbolic proofs + planned neural layer) -- `PLAYBOOK.scm` — Operational runbook, build procedures, incident response -- `HANDOVER.scm` — Legacy handover artefact (historical) -- `ROADMAP.scm` — Unified roadmap across all lithoglyph subprojects - -## Language Policy - -### ALLOWED -- **Forth** — Storage kernel (gforth) -- **Zig** — Bridge, BEAM NIF, API layer -- **Idris2** — ABI proofs (dependent types) -- **Lean 4** — Normalization proofs, GQL-DT -- **Factor** — GQL runtime (concatenative) -- **Elixir** — OTP control plane, HTTP API -- **Rust** — BEAM NIF (Rustler), studio backend -- **ReScript** — Client libraries (if needed) -- **Nickel** — Configuration - -### BANNED -- TypeScript, Python, Go, Java, Node.js, npm - -## Build Commands - -### Core Components (dependency order) -```bash -# 1. Forth kernel (17 tests) -cd core-forth && gforth test/lithoglyph-tests.fs - -# 2. Zig bridge (primary — all tests) -cd core-zig && zig build test - -# 3. Zig FFI delegation (delegates to core-zig) -cd ffi/zig && zig build test - -# 4. Idris2 ABI type-check (3 files, must be clean) -idris2 --source-dir src --check src/Lith/LithBridge.idr -idris2 --source-dir src --check src/Lith/LithLayout.idr -idris2 --source-dir src --check src/Lith/LithForeign.idr - -# 5. Lean 4 normalizer (52 proofs) -cd normalizer && lake build - -# 6. Factor runtime -cd core-factor && factor -run=listener # manual verification -``` - -### Satellites -```bash -# BEAM NIF (Rust — 0 warnings required) -cd beam/native_rust && cargo build - -# BEAM NIF (Zig) -cd beam/native && zig build - -# Elixir HTTP (lith-http) -cd lith-http && mix deps.get && mix compile && mix test - -# GQL-DT (Lean 4) -cd gql-dt && lake build - -# Studio (Gossamer — mostly mock data) -cd studio && cargo build -``` - -### Quick Verification -```bash -just test # core-zig + ffi/zig + core-forth -just build-all # everything in dependency order -``` - -## Component Status - -| Component | Build | Tests | Notes | -|-----------|-------|-------|-------| -| core-forth | PASS | 17/17 | Block storage, journal, data model | -| core-zig | PASS | PASS | 19 real functions, WAL commit | -| ffi/zig | PASS | PASS | Delegates to core-zig | -| Idris2 ABI | PASS | N/A | 3 files type-check clean, 0 believe_me | -| Lean 4 normalizer | PASS | 52 PASS | FD discovery proofs | -| core-factor | PASS | N/A | GQL parser/planner/executor | -| BEAM NIF (Zig) | PASS | N/A | Real FFI calls | -| BEAM NIF (Rust) | PASS | N/A | Rustler 0.35, 0 warnings | -| lith-http | PASS | PASS | M15 complete | -| gql-dt | PASS | claims 100% | Needs honest audit | -| glyphbase | ? | ? | Needs honest audit | -| api (Zig) | BROKEN | N/A | 83 old std.http.Server call sites | -| studio | PASS | N/A | 11 Gossamer commands return mock data | - -## Patterns and Anti-Patterns - -### Banned Patterns (CI enforced) -- `believe_me` in Idris2 — use `%foreign prim__callbackToAnyPtr` pattern instead -- `assert_total`, `assert_smaller`, `unsafePerformIO` in Idris2 -- `sorry` in Lean 4 -- `Admitted` in Coq (if any) -- Zig unsafe casts without `// SAFETY:` annotation -- SQL string concatenation — use parameterized query builders -- Hardcoded secrets — use env vars with `${VAR:-}` defaults - -### Key Design Patterns -- **Bridge delegation**: `ffi/zig/` → `core-zig/` (never reverse) -- **ABI-first**: Idris2 proofs define the interface contract, Zig implements it -- **Proof-verified schemas**: Lean 4 normalizer proves schema correctness -- **Narrative provenance**: Every mutation has a story — who, what, when, why -- **WAL-first writes**: All mutations go through Write-Ahead Log before blocks - -## Related Projects - -- **verisimdb** — Octad database sibling (shares GQL patterns) -- **quandledb** — Knot-theoretic database sibling -- **nqc** — Normal-form Query Compiler -- **proven** — Formally verified safety library (SafeString, SafeJson, etc.) -- **hypatia** — Neurosymbolic CI/CD scanner -- **gitbot-fleet** — Bot orchestration (rhodibot, echidnabot, sustainabot) - -## Known Issues - -- **API layer**: `api/src/rest.zig` and `api/src/grpc.zig` use old `std.http.Server` API — 83 call sites need Zig 0.15.2 migration -- **Studio**: 11 Tauri commands return mock data, not real bridge calls -- **IP rename**: `Lith`/`Form` namespace must be renamed before public release -- **Idris2 Nat reduction**: Proof type signatures must use concrete literals, not function-defined constants (Idris2 0.8 limitation) -- **gql-dt/glyphbase**: Claim 100% completion — need honest audit diff --git a/lithoglyph/.editorconfig b/lithoglyph/.editorconfig deleted file mode 100644 index 1e8edcb2..00000000 --- a/lithoglyph/.editorconfig +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 2 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -trim_trailing_whitespace = false - -[*.adoc] -trim_trailing_whitespace = false - -[*.rs] -indent_size = 4 - -[*.zig] -indent_size = 4 - -[*.ada] -indent_size = 3 - -[*.adb] -indent_size = 3 - -[*.ads] -indent_size = 3 - -[*.ex] -indent_size = 2 - -[*.exs] -indent_size = 2 - -[*.hs] -indent_size = 2 - -[*.res] -indent_size = 2 - -[*.resi] -indent_size = 2 - -[*.ncl] -indent_size = 2 - -[*.scm] -indent_size = 2 - -[*.nix] -indent_size = 2 - -[Justfile] -indent_style = space -indent_size = 4 - -[justfile] -indent_style = space -indent_size = 4 - -[Makefile] -indent_style = tab diff --git a/lithoglyph/.gitattributes b/lithoglyph/.gitattributes deleted file mode 100644 index dd743cd1..00000000 --- a/lithoglyph/.gitattributes +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitattributes - -* text=auto eol=lf - -# Source -*.rs text eol=lf diff=rust -*.ex text eol=lf diff=elixir -*.exs text eol=lf diff=elixir -*.jl text eol=lf -*.res text eol=lf -*.resi text eol=lf -*.ada text eol=lf diff=ada -*.adb text eol=lf diff=ada -*.ads text eol=lf diff=ada -*.hs text eol=lf -*.chpl text eol=lf -*.scm text eol=lf -*.ncl text eol=lf -*.nix text eol=lf -*.zig text eol=lf - -# Docs -*.md text eol=lf diff=markdown -*.adoc text eol=lf -*.txt text eol=lf - -# Data -*.json text eol=lf -*.yaml text eol=lf -*.yml text eol=lf -*.toml text eol=lf - -# Config -.gitignore text eol=lf -.gitattributes text eol=lf -.editorconfig text eol=lf -.tool-versions text eol=lf -justfile text eol=lf -Makefile text eol=lf -Containerfile text eol=lf - -# Scripts -*.sh text eol=lf - -# Binary -*.png binary -*.jpg binary -*.gif binary -*.pdf binary -*.woff2 binary -*.zip binary -*.gz binary - -# Lock files -Cargo.lock text eol=lf -diff -flake.lock text eol=lf -diff diff --git a/lithoglyph/.github/ISSUE_TEMPLATE/bug_report.md b/lithoglyph/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index dd84ea78..00000000 --- a/lithoglyph/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. diff --git a/lithoglyph/.github/ISSUE_TEMPLATE/custom.md b/lithoglyph/.github/ISSUE_TEMPLATE/custom.md deleted file mode 100644 index 48d5f81f..00000000 --- a/lithoglyph/.github/ISSUE_TEMPLATE/custom.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Custom issue template -about: Describe this issue template's purpose here. -title: '' -labels: '' -assignees: '' - ---- - - diff --git a/lithoglyph/.github/ISSUE_TEMPLATE/feature_request.md b/lithoglyph/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index bbcbbe7d..00000000 --- a/lithoglyph/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/lithoglyph/.github/workflows/casket-pages.yml b/lithoglyph/.github/workflows/casket-pages.yml deleted file mode 100644 index f6d49797..00000000 --- a/lithoglyph/.github/workflows/casket-pages.yml +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: GitHub Pages - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Checkout casket-ssg - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - repository: hyperpolymath/casket-ssg - path: .casket-ssg - - - name: Setup GHCup - uses: haskell-actions/setup@ec49483bfc012387b227434aba94f59a6ecd0900 # v2 - with: - ghc-version: '9.8.2' - cabal-version: '3.10' - - - name: Cache Cabal - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: | - ~/.cabal/packages - ~/.cabal/store - .casket-ssg/dist-newstyle - key: ${{ runner.os }}-casket-${{ hashFiles('.casket-ssg/casket-ssg.cabal') }} - - - name: Build casket-ssg - working-directory: .casket-ssg - run: cabal build - - - name: Build site - run: | - mkdir -p site _site - # Generate index.md from README if site/index.md doesn't exist - if [ ! -f site/index.md ]; then - if [ -f README.adoc ]; then - # Convert AsciiDoc to Markdown (basic conversion) - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - cat README.adoc >> site/index.md - elif [ -f README.md ]; then - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - cat README.md >> site/index.md - else - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - echo "" >> site/index.md - echo "# $(basename $PWD)" >> site/index.md - echo "" >> site/index.md - echo "Documentation coming soon." >> site/index.md - fi - fi - cd .casket-ssg && cabal run casket-ssg -- build ../site ../_site - - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 - with: - path: '_site' - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/lithoglyph/.github/workflows/ci.yml b/lithoglyph/.github/workflows/ci.yml deleted file mode 100644 index 31a2970b..00000000 --- a/lithoglyph/.github/workflows/ci.yml +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Lithoglyph CI — Build and test all components -# -# Runs on push to main and all pull requests. -# Tests: Zig core, Forth blocks, C FFI integration. - -name: CI - -on: - push: - branches: [main] - pull_request: - branches: [main] - -permissions: read-all - -jobs: - # =========================================================================== - # Job 1: Build and test core-zig - # =========================================================================== - build-zig: - name: Build & Test core-zig - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Build core-zig - working-directory: core-zig - run: zig build - - - name: Run core-zig tests - working-directory: core-zig - run: zig build test - - - name: Upload Zig build artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: zig-build - path: | - core-zig/zig-out/lib/ - retention-days: 1 - - # =========================================================================== - # Job 2: Forth block tests - # =========================================================================== - test-forth: - name: Forth Block Tests - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Install gforth - run: sudo apt-get update && sudo apt-get install -y gforth - - - name: Run Forth block tests - working-directory: core-forth/test - run: gforth test-blocks.fs -e bye - - - name: Verify Forth source files load - working-directory: core-forth/src - run: | - gforth lithoglyph-blocks.fs -e 'bye' - gforth lithoglyph-journal.fs -e 'bye' - gforth lithoglyph-model.fs -e 'bye' - - # =========================================================================== - # Job 3: C FFI integration tests (depends on Zig build) - # =========================================================================== - test-ffi: - name: C FFI Integration Tests - runs-on: ubuntu-latest - needs: build-zig - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Build Zig libraries - working-directory: core-zig - run: zig build - - - name: Compile C FFI integration tests - working-directory: core-zig - run: | - gcc -I../generated/abi \ - -o test-ffi-integration \ - test-ffi-integration.c \ - -Lzig-out/lib \ - -llith_bridge - - - name: Run FFI integration tests - working-directory: core-zig - run: | - LD_LIBRARY_PATH=zig-out/lib ./test-ffi-integration - - - name: Verify ABI exports - working-directory: core-zig - run: | - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_db_open' - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_apply' - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_version' - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_introspect_schema' diff --git a/lithoglyph/.github/workflows/codeql.yml b/lithoglyph/.github/workflows/codeql.yml deleted file mode 100644 index 50b98c88..00000000 --- a/lithoglyph/.github/workflows/codeql.yml +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: CodeQL Security Analysis - -on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - schedule: - - cron: '0 6 * * 1' - -permissions: read-all - -jobs: - analyze: - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - strategy: - fail-fast: false - matrix: - include: - - language: javascript-typescript - build-mode: none - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Initialize CodeQL - uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.28.1 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.28.1 - with: - category: "/language:${{ matrix.language }}" diff --git a/lithoglyph/.github/workflows/governance.yml b/lithoglyph/.github/workflows/governance.yml deleted file mode 100644 index b0b1ed6d..00000000 --- a/lithoglyph/.github/workflows/governance.yml +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# governance.yml — single wrapper calling the shared estate governance bundle -# in hyperpolymath/standards instead of carrying per-repo copies. -# -# Replaces the per-repo governance scaffolding removed in the same commit: -# quality.yml, guix-nix-policy.yml, npm-bun-blocker.yml, ts-blocker.yml, -# security-policy.yml, rsr-antipattern.yml, wellknown-enforcement.yml, -# workflow-linter.yml -# -# Load-bearing build/security workflows stay standalone in the repo -# (rust-ci, codeql, dependabot, release, scan/mirror/pages plumbing). - -name: Governance - -on: - push: - branches: [main, master] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - governance: - uses: hyperpolymath/standards/.github/workflows/governance-reusable.yml@main diff --git a/lithoglyph/.github/workflows/hypatia-scan.yml b/lithoglyph/.github/workflows/hypatia-scan.yml deleted file mode 100644 index 5b59919d..00000000 --- a/lithoglyph/.github/workflows/hypatia-scan.yml +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Hypatia Neurosymbolic CI/CD Security Scan -name: Hypatia Security Scan - -on: - push: - branches: [ main, master, develop ] - pull_request: - branches: [ main, master ] - schedule: - - cron: '0 0 * * 0' # Weekly on Sunday - workflow_dispatch: - -permissions: read-all - -jobs: - scan: - name: Hypatia Neurosymbolic Analysis - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 # Full history for better pattern analysis - - - name: Setup Elixir for Hypatia scanner - uses: erlef/setup-beam@2f0cc07b4b9bea248ae098aba9e1a8a1de5ec24c # v1.18.2 - with: - elixir-version: '1.19.4' - otp-version: '28.3' - - - name: Clone Hypatia - run: | - if [ ! -d "$HOME/hypatia" ]; then - git clone https://github.com/hyperpolymath/hypatia.git "$HOME/hypatia" - fi - - - name: Build Hypatia scanner (if needed) - working-directory: ${{ env.HOME }}/hypatia - run: | - if [ ! -f hypatia-v2 ]; then - echo "Building hypatia-v2 scanner..." - cd scanner - mix deps.get - mix escript.build - mv hypatia ../hypatia-v2 - fi - - - name: Run Hypatia scan - id: scan - run: | - echo "Scanning repository: ${{ github.repository }}" - - # Run scanner - HYPATIA_FORMAT=json "$HOME/hypatia/hypatia-cli.sh" scan . > hypatia-findings.json - - # Count findings - FINDING_COUNT=$(jq '. | length' hypatia-findings.json 2>/dev/null || echo 0) - echo "findings_count=$FINDING_COUNT" >> $GITHUB_OUTPUT - - # Extract severity counts - CRITICAL=$(jq '[.[] | select(.severity == "critical")] | length' hypatia-findings.json) - HIGH=$(jq '[.[] | select(.severity == "high")] | length' hypatia-findings.json) - MEDIUM=$(jq '[.[] | select(.severity == "medium")] | length' hypatia-findings.json) - - echo "critical=$CRITICAL" >> $GITHUB_OUTPUT - echo "high=$HIGH" >> $GITHUB_OUTPUT - echo "medium=$MEDIUM" >> $GITHUB_OUTPUT - - echo "## Hypatia Scan Results" >> $GITHUB_STEP_SUMMARY - echo "- Total findings: $FINDING_COUNT" >> $GITHUB_STEP_SUMMARY - echo "- Critical: $CRITICAL" >> $GITHUB_STEP_SUMMARY - echo "- High: $HIGH" >> $GITHUB_STEP_SUMMARY - echo "- Medium: $MEDIUM" >> $GITHUB_STEP_SUMMARY - - - name: Upload findings artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: hypatia-findings - path: hypatia-findings.json - retention-days: 90 - - - name: Submit findings to gitbot-fleet (Phase 2) - if: steps.scan.outputs.findings_count > 0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_SHA: ${{ github.sha }} - run: | - echo "📤 Submitting ${{ steps.scan.outputs.findings_count }} findings to gitbot-fleet..." - - # Clone gitbot-fleet to temp directory - FLEET_DIR="/tmp/gitbot-fleet-$$" - git clone https://github.com/hyperpolymath/gitbot-fleet.git "$FLEET_DIR" - - # Run submission script - bash "$FLEET_DIR/scripts/submit-finding.sh" hypatia-findings.json - - # Cleanup - rm -rf "$FLEET_DIR" - - echo "✅ Finding submission complete" - - - name: Check for critical issues - if: steps.scan.outputs.critical > 0 - run: | - echo "⚠️ Critical security issues found!" - echo "Review hypatia-findings.json for details" - # Don't fail the build yet - just warn - # exit 1 - - - name: Generate scan report - run: | - cat << EOF > hypatia-report.md - # Hypatia Security Scan Report - - **Repository:** ${{ github.repository }} - **Scan Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC") - **Commit:** ${{ github.sha }} - - ## Summary - - | Severity | Count | - |----------|-------| - | Critical | ${{ steps.scan.outputs.critical }} | - | High | ${{ steps.scan.outputs.high }} | - | Medium | ${{ steps.scan.outputs.medium }} | - | **Total**| ${{ steps.scan.outputs.findings_count }} | - - ## Next Steps - - 1. Review findings in the artifact: hypatia-findings.json - 2. Auto-fixable issues will be addressed by robot-repo-automaton (Phase 3) - 3. Manual review required for complex issues - - ## Learning - - These findings feed Hypatia's learning engine to improve future rules. - - --- - *Powered by [Hypatia](https://github.com/hyperpolymath/hypatia) - Neurosymbolic CI/CD Intelligence* - EOF - - cat hypatia-report.md >> $GITHUB_STEP_SUMMARY - - - name: Comment on PR with findings - if: github.event_name == 'pull_request' && steps.scan.outputs.findings_count > 0 - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const fs = require('fs'); - const findings = JSON.parse(fs.readFileSync('hypatia-findings.json', 'utf8')); - - const critical = findings.filter(f => f.severity === 'critical').length; - const high = findings.filter(f => f.severity === 'high').length; - - let comment = `## 🔍 Hypatia Security Scan\n\n`; - comment += `**Findings:** ${findings.length} issues detected\n\n`; - comment += `| Severity | Count |\n|----------|-------|\n`; - comment += `| 🔴 Critical | ${critical} |\n`; - comment += `| 🟠 High | ${high} |\n`; - comment += `| 🟡 Medium | ${findings.length - critical - high} |\n\n`; - - if (critical > 0) { - comment += `⚠️ **Action Required:** Critical security issues found!\n\n`; - } - - comment += `
View findings\n\n`; - comment += `\`\`\`json\n${JSON.stringify(findings.slice(0, 10), null, 2)}\n\`\`\`\n`; - comment += `
\n\n`; - comment += `*Powered by Hypatia Neurosymbolic CI/CD Intelligence*`; - - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: comment - }); diff --git a/lithoglyph/.github/workflows/instant-sync.yml b/lithoglyph/.github/workflows/instant-sync.yml deleted file mode 100644 index 228dc438..00000000 --- a/lithoglyph/.github/workflows/instant-sync.yml +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Instant Forge Sync - Triggers propagation to all forges on push/release -name: Instant Sync - -on: - push: - branches: [main, master] - release: - types: [published] - -permissions: - contents: read - -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - name: Trigger Propagation - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v3 - with: - token: ${{ secrets.FARM_DISPATCH_TOKEN }} - repository: hyperpolymath/.git-private-farm - event-type: propagate - client-payload: |- - { - "repo": "${{ github.event.repository.name }}", - "ref": "${{ github.ref }}", - "sha": "${{ github.sha }}", - "forges": "" - } - - - name: Confirm - run: echo "::notice::Propagation triggered for ${{ github.event.repository.name }}" diff --git a/lithoglyph/.github/workflows/jekyll-gh-pages.yml b/lithoglyph/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index 98b5a255..00000000 --- a/lithoglyph/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages dependencies preinstalled - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Build job - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1 - with: - source: ./ - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v4 - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/lithoglyph/.github/workflows/jekyll.yml b/lithoglyph/.github/workflows/jekyll.yml deleted file mode 100644 index 75d16575..00000000 --- a/lithoglyph/.github/workflows/jekyll.yml +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll site to Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Build job - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Setup Ruby - # https://github.com/ruby/setup-ruby/releases/tag/v1.207.0 - uses: ruby/setup-ruby@708024e6c902387ab41de36e1669e43b5ee7085e - with: - ruby-version: '3.1' # Not needed with a .ruby-version file - bundler-cache: true # runs 'bundle install' and caches installed gems automatically - cache-version: 0 # Increment this number if you need to re-download cached gems - - name: Setup Pages - id: pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - - name: Build with Jekyll - # Outputs to the './_site' directory by default - run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}" - env: - JEKYLL_ENV: production - - name: Upload artifact - # Automatically uploads an artifact from the './_site' directory by default - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v4 - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/lithoglyph/.github/workflows/mirror.yml b/lithoglyph/.github/workflows/mirror.yml deleted file mode 100644 index 90011ad3..00000000 --- a/lithoglyph/.github/workflows/mirror.yml +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2025 Jonathan D.A. Jewell -name: Mirror to Git Forges - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: read-all - -jobs: - mirror-gitlab: - runs-on: ubuntu-latest - if: vars.GITLAB_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_KEY }} - - - name: Mirror to GitLab - run: | - ssh-keyscan -t ed25519 gitlab.com >> ~/.ssh/known_hosts - git remote add gitlab git@gitlab.com:hyperpolymath/${{ github.event.repository.name }}.git || true - git push --force gitlab main - - mirror-bitbucket: - runs-on: ubuntu-latest - if: vars.BITBUCKET_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.BITBUCKET_SSH_KEY }} - - - name: Mirror to Bitbucket - run: | - ssh-keyscan -t ed25519 bitbucket.org >> ~/.ssh/known_hosts - git remote add bitbucket git@bitbucket.org:hyperpolymath/${{ github.event.repository.name }}.git || true - git push --force bitbucket main - - mirror-codeberg: - runs-on: ubuntu-latest - if: vars.CODEBERG_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.CODEBERG_SSH_KEY }} - - - name: Mirror to Codeberg - run: | - ssh-keyscan -t ed25519 codeberg.org >> ~/.ssh/known_hosts - git remote add codeberg git@codeberg.org:hyperpolymath/${{ github.event.repository.name }}.git || true - git push --force codeberg main - - mirror-sourcehut: - runs-on: ubuntu-latest - if: vars.SOURCEHUT_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.SOURCEHUT_SSH_KEY }} - - - name: Mirror to SourceHut - run: | - ssh-keyscan -t ed25519 git.sr.ht >> ~/.ssh/known_hosts - git remote add sourcehut git@git.sr.ht:~hyperpolymath/${{ github.event.repository.name }} || true - git push --force sourcehut main - - mirror-disroot: - runs-on: ubuntu-latest - if: vars.DISROOT_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.DISROOT_SSH_KEY }} - - - name: Mirror to Disroot - run: | - ssh-keyscan -t ed25519 git.disroot.org >> ~/.ssh/known_hosts - git remote add disroot git@git.disroot.org:hyperpolymath/${{ github.event.repository.name }}.git || true - git push --force disroot main - - mirror-gitea: - runs-on: ubuntu-latest - if: vars.GITEA_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 - with: - ssh-private-key: ${{ secrets.GITEA_SSH_KEY }} - - - name: Mirror to Gitea - run: | - ssh-keyscan -t ed25519 ${{ vars.GITEA_HOST }} >> ~/.ssh/known_hosts - git remote add gitea git@${{ vars.GITEA_HOST }}:hyperpolymath/${{ github.event.repository.name }}.git || true - git push --force gitea main - - mirror-radicle: - runs-on: ubuntu-latest - if: vars.RADICLE_MIRROR_ENABLED == 'true' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # stable - with: - toolchain: stable - - - name: Install Radicle - run: | - # Install via cargo (safer than curl|sh) - cargo install radicle-cli --locked - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Mirror to Radicle - run: | - echo "${{ secrets.RADICLE_KEY }}" > ~/.radicle/keys/radicle - chmod 600 ~/.radicle/keys/radicle - rad sync --announce || echo "Radicle sync attempted" diff --git a/lithoglyph/.github/workflows/scorecard-enforcer.yml b/lithoglyph/.github/workflows/scorecard-enforcer.yml deleted file mode 100644 index e7c897d5..00000000 --- a/lithoglyph/.github/workflows/scorecard-enforcer.yml +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Prevention workflow - runs OpenSSF Scorecard and fails on low scores -name: OpenSSF Scorecard Enforcer - -on: - push: - branches: [main] - schedule: - - cron: '0 6 * * 1' # Weekly on Monday - workflow_dispatch: - -permissions: read-all - -jobs: - scorecard: - runs-on: ubuntu-latest - permissions: - security-events: write - id-token: write # For OIDC - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4 - with: - persist-credentials: false - - - name: Run Scorecard - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 - with: - results_file: results.sarif - results_format: sarif - publish_results: true - - - name: Upload SARIF - uses: github/codeql-action/upload-sarif@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v3 - with: - sarif_file: results.sarif - - - name: Check minimum score - run: | - # Parse score from results - SCORE=$(jq -r '.runs[0].tool.driver.properties.score // 0' results.sarif 2>/dev/null || echo "0") - - echo "OpenSSF Scorecard Score: $SCORE" - - # Minimum acceptable score (0-10 scale) - MIN_SCORE=5 - - if [ "$(echo "$SCORE < $MIN_SCORE" | bc -l)" = "1" ]; then - echo "::error::Scorecard score $SCORE is below minimum $MIN_SCORE" - exit 1 - fi - - # Check specific high-priority items - check-critical: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4 - - - name: Check SECURITY.md exists - run: | - if [ ! -f "SECURITY.md" ]; then - echo "::error::SECURITY.md is required" - exit 1 - fi - - - name: Check for pinned dependencies - run: | - # Check workflows for unpinned actions - unpinned=$(grep -r "uses:.*@v[0-9]" .github/workflows/*.yml 2>/dev/null | grep -v "#" | head -5 || true) - if [ -n "$unpinned" ]; then - echo "::warning::Found unpinned actions:" - echo "$unpinned" - fi diff --git a/lithoglyph/.github/workflows/scorecard.yml b/lithoglyph/.github/workflows/scorecard.yml deleted file mode 100644 index 476ed3f0..00000000 --- a/lithoglyph/.github/workflows/scorecard.yml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: OSSF Scorecard -on: - push: - branches: [main, master] - schedule: - - cron: '0 4 * * *' - workflow_dispatch: - -permissions: read-all - -jobs: - analysis: - runs-on: ubuntu-latest - permissions: - security-events: write - id-token: write - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Run Scorecard - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - - - name: Upload results - uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.31.8 - with: - sarif_file: results.sarif diff --git a/lithoglyph/.github/workflows/secret-scanner.yml b/lithoglyph/.github/workflows/secret-scanner.yml deleted file mode 100644 index b2dcc860..00000000 --- a/lithoglyph/.github/workflows/secret-scanner.yml +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Prevention workflow - scans for hardcoded secrets before they reach main -name: Secret Scanner - -on: - pull_request: - push: - branches: [main] - -permissions: read-all - -jobs: - trufflehog: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 # Full history for scanning - - - name: TruffleHog Secret Scan - uses: trufflesecurity/trufflehog@8a8ef8526528d8a4ff3e2c90be08e25ef8efbd9b # v3 - with: - extra_args: --only-verified --fail - - gitleaks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 - - - name: Gitleaks Secret Scan - uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Rust-specific: Check for hardcoded crypto values - rust-secrets: - runs-on: ubuntu-latest - if: hashFiles('**/Cargo.toml') != '' - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Check for hardcoded secrets in Rust - run: | - # Patterns that suggest hardcoded secrets - PATTERNS=( - 'const.*SECRET.*=.*"' - 'const.*KEY.*=.*"[a-zA-Z0-9]{16,}"' - 'const.*TOKEN.*=.*"' - 'let.*api_key.*=.*"' - 'HMAC.*"[a-fA-F0-9]{32,}"' - 'password.*=.*"[^"]+"' - ) - - found=0 - for pattern in "${PATTERNS[@]}"; do - if grep -rn --include="*.rs" -E "$pattern" src/; then - echo "WARNING: Potential hardcoded secret found matching: $pattern" - found=1 - fi - done - - if [ $found -eq 1 ]; then - echo "::error::Potential hardcoded secrets detected. Use environment variables instead." - exit 1 - fi diff --git a/lithoglyph/.github/workflows/test-suite.yml b/lithoglyph/.github/workflows/test-suite.yml deleted file mode 100644 index 2e3c977f..00000000 --- a/lithoglyph/.github/workflows/test-suite.yml +++ /dev/null @@ -1,168 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph - Full Test Suite -# -# Runs Forth, Zig, C FFI, Lean, and ReScript tests. -# Complements zig-tests.yml (which focuses on multiversion Zig testing). - -name: Test Suite - -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main, develop ] - -permissions: read-all - -jobs: - test-forth: - name: Forth Block Tests - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Install gforth - run: sudo apt-get update && sudo apt-get install -y gforth - - - name: Run block tests - working-directory: core-forth/test - run: | - echo "=== Forth Block Storage Tests ===" - gforth test-blocks.fs -e bye - - - name: Check Forth source loads - working-directory: core-forth/src - run: | - echo "=== Loading Forth source files ===" - gforth lithoglyph-blocks.fs -e 'bye' - gforth lithoglyph-journal.fs -e 'bye' - gforth lithoglyph-model.fs -e 'bye' - echo "All Forth files load successfully" - - test-zig-core: - name: Zig Core Tests - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Run unit tests - working-directory: core-zig - run: | - echo "=== Zig Unit Tests ===" - zig build test - echo "All Zig tests passed" - - - name: Build libraries - working-directory: core-zig - run: | - echo "=== Building Libraries ===" - zig build - ls -lh zig-out/lib/ - - test-ffi: - name: C FFI Integration Tests - runs-on: ubuntu-latest - needs: test-zig-core - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Build Zig libraries - working-directory: core-zig - run: zig build - - - name: Compile C FFI tests - working-directory: core-zig - run: | - gcc -o test-version-only test-version-only.c -L zig-out/lib -llith_bridge - gcc -o test-db-open test-db-open.c -L zig-out/lib -llith_bridge - gcc -o test-ffi-integration test-ffi-integration.c -L zig-out/lib -llith_bridge - - - name: Run FFI tests - working-directory: core-zig - run: | - echo "=== C FFI Integration Tests ===" - LD_LIBRARY_PATH=zig-out/lib ./test-version-only - LD_LIBRARY_PATH=zig-out/lib ./test-db-open - LD_LIBRARY_PATH=zig-out/lib ./test-ffi-integration - echo "All FFI tests passed" - - - name: Verify ABI exports - working-directory: core-zig - run: | - echo "=== ABI Export Check ===" - nm -D zig-out/lib/liblith_bridge.so | grep ' T lith_' | sort - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_db_open' - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_apply' - nm -D zig-out/lib/liblith_bridge.so | grep -q 'lith_introspect_schema' - echo "All expected ABI exports present" - - test-lean: - name: Lean 4 Normalizer Tests - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Install elan (Lean version manager) - run: | - curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain none - echo "$HOME/.elan/bin" >> $GITHUB_PATH - - - name: Build and test Lean project - working-directory: normalizer/lean - run: | - echo "=== Lean 4 Normalizer ===" - lake build - echo "Lean build successful (tests run via #eval during build)" - - test-rescript: - name: ReScript Test Suites - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Deno - uses: denoland/setup-deno@5fae568d37c3b73e63e2911f55b8b3fbc76d26f8 # v2.0.2 - with: - deno-version: v2.x - - - name: Run property tests - working-directory: tests/property - run: | - echo "=== Property-Based Tests ===" - deno task test || echo "Property tests need ReScript compilation (skipping for now)" - - - name: Run fuzz tests (quick) - working-directory: tests/fuzz - run: | - echo "=== Fuzz Tests (Quick) ===" - deno task fuzz:quick || echo "Fuzz tests need ReScript compilation (skipping for now)" diff --git a/lithoglyph/.github/workflows/zig-tests.yml b/lithoglyph/.github/workflows/zig-tests.yml deleted file mode 100644 index 20fbd4b1..00000000 --- a/lithoglyph/.github/workflows/zig-tests.yml +++ /dev/null @@ -1,279 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph - Zig Component Tests -# -# Tests the core Zig bridge and block storage layer. -# Runs on every push and PR to ensure industrial-grade quality. - -name: Zig Tests - -on: - push: - branches: [ main, develop ] - paths: - - 'core-zig/**' - - '.github/workflows/zig-tests.yml' - pull_request: - branches: [ main, develop ] - paths: - - 'core-zig/**' - - '.github/workflows/zig-tests.yml' - -permissions: - contents: read - -jobs: - test-zig-components: - name: Test Zig Components - runs-on: ubuntu-latest - - strategy: - matrix: - zig-version: ['0.13.0', '0.14.0', '0.15.2'] - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: ${{ matrix.zig-version }} - - - name: Verify Zig installation - run: | - zig version - zig env - - - name: Run block storage tests - working-directory: core-zig - run: | - echo "=== Testing Block Storage Layer ===" - zig test src/blocks.zig - - echo "" - echo "✅ Block storage tests passed" - - - name: Run bridge tests - working-directory: core-zig - run: | - echo "=== Testing Bridge Layer ===" - zig test src/bridge.zig - - echo "" - echo "✅ Bridge tests passed" - - - name: Build static library - working-directory: core-zig - run: | - echo "=== Building Static Library ===" - zig build-lib src/bridge.zig -O ReleaseSafe - ls -lh libbridge.a - - echo "" - echo "✅ Static library built successfully" - - - name: Build shared library - working-directory: core-zig - run: | - echo "=== Building Shared Library ===" - zig build-lib -dynamic src/bridge.zig -O ReleaseSafe - ls -lh libbridge.so - - echo "" - echo "✅ Shared library built successfully" - - - name: Check for memory leaks (test mode) - working-directory: core-zig - run: | - echo "=== Memory Leak Detection ===" - # Run tests with safety checks enabled - zig test src/blocks.zig -O Debug - - echo "" - echo "✅ No memory leaks detected" - - - name: Verify ABI compatibility - working-directory: core-zig - run: | - echo "=== ABI Compatibility Check ===" - - # Generate symbol list - zig build-lib src/bridge.zig -O ReleaseFast - nm -D libbridge.so | grep "lith_" | head -20 - - # Verify key exports - if nm -D libbridge.so | grep -q "lith_db_open"; then - echo "✅ lith_db_open exported" - else - echo "❌ lith_db_open missing" - exit 1 - fi - - if nm -D libbridge.so | grep -q "lith_apply"; then - echo "✅ lith_apply exported" - else - echo "❌ lith_apply missing" - exit 1 - fi - - echo "" - echo "✅ ABI exports verified" - - test-block-formats: - name: Test Block Format Compatibility - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Test block format (64-byte headers) - working-directory: core-zig - run: | - cat > test_format.zig << 'EOF' -const std = @import("std"); -const blocks = @import("src/blocks.zig"); - -test "block header is exactly 64 bytes" { - try std.testing.expectEqual(@as(usize, 64), @sizeOf(blocks.BlockHeader)); -} - -test "block is exactly 4096 bytes" { - try std.testing.expectEqual(@as(usize, 4096), @sizeOf(blocks.Block)); -} - -test "magic bytes match Forth spec" { - try std.testing.expectEqual(@as(u32, 0x4C474800), blocks.BLOCK_MAGIC); -} -EOF - - zig test test_format.zig - rm test_format.zig - - echo "✅ Block format verified (Forth-compatible)" - - test-crc32c-correctness: - name: Test CRC32C Implementation - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Verify CRC32C against test vectors - working-directory: core-zig - run: | - cat > test_crc32c.zig << 'EOF' -const std = @import("std"); -const blocks = @import("src/blocks.zig"); - -// Test vectors from RFC 3720 (iSCSI CRC32C) -test "CRC32C test vectors" { - // Empty string - const empty = ""; - try std.testing.expectEqual(@as(u32, 0), blocks.crc32c(empty, 0)); - - // Known test patterns - const test1 = "123456789"; - const crc1 = blocks.crc32c(test1, test1.len); - try std.testing.expect(crc1 != 0); - - const test2 = "hello world"; - const crc2 = blocks.crc32c(test2, test2.len); - try std.testing.expect(crc2 != 0); - - // Different inputs produce different CRCs - try std.testing.expect(crc1 != crc2); -} -EOF - - zig test test_crc32c.zig - rm test_crc32c.zig - - echo "✅ CRC32C implementation verified" - - integration-test: - name: Integration Test (Block Storage) - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.1 - with: - version: '0.15.2' - - - name: Create integration test - working-directory: core-zig - run: | - cat > integration_test.zig << 'EOF' -const std = @import("std"); -const blocks = @import("src/blocks.zig"); - -test "full storage workflow" { - const allocator = std.testing.allocator; - const path = "integration_test.lgh"; - defer std.fs.cwd().deleteFile(path) catch {}; - - // Create database - const storage = try blocks.BlockStorage.open(allocator, path); - defer storage.deinit(); - - // Allocate and write 10 document blocks - var i: usize = 0; - while (i < 10) : (i += 1) { - const block_id = try storage.allocateBlock(.document); - var block = try storage.readBlock(block_id); - - var buf: [100]u8 = undefined; - const content = try std.fmt.bufPrint(&buf, "Document {d}", .{i}); - try block.setPayload(content); - - try storage.writeBlock(block_id, &block); - } - - // Verify we can read them back - i = 1; - while (i <= 10) : (i += 1) { - const block = try storage.readBlock(i); - try std.testing.expect(block.getPayload().len > 0); - } - - // Append journal entries - _ = try storage.appendJournal("Operation 1"); - _ = try storage.appendJournal("Operation 2"); - - // Verify superblock updated - try std.testing.expect(storage.superblock.block_count > 10); -} -EOF - - zig test integration_test.zig - rm integration_test.zig - - echo "✅ Integration test passed" - - check-todos: - name: Check TODOs in Zig Code - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Find TODOs - run: | - echo "=== TODOs in Zig code ===" - grep -rn "TODO\|FIXME\|XXX" core-zig/src/ --include="*.zig" || echo "No TODOs found ✅" diff --git a/lithoglyph/.gitignore b/lithoglyph/.gitignore deleted file mode 100644 index 3b60dadf..00000000 --- a/lithoglyph/.gitignore +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph .gitignore - -# Build outputs -/target/ -/build/ -/dist/ -*.o -*.a -*.so -*.dylib - -# Forth -*.fs~ -*.4th~ - -# Zig (all cache locations) -zig-cache/ -zig-out/ -.zig-cache/ - -# Factor -.factor-rc -.factor-boot-rc - -# Elixir -/_build/ -/deps/ -/*.ez -/cover/ -/doc/ -/.fetch -erl_crash.dump -*.beam - -# Editor -*~ -*.swp -*.swo -.idea/ -.vscode/ -*.sublime-* - -# OS -.DS_Store -Thumbs.db - -# Test artifacts -/test-output/ -*.log -*.fdb - -# Compiled test binaries -core-zig/test-ffi-integration -core-zig/test-db-open -core-zig/test-version-only - -# Database files (runtime) -*.lgh - -# Lean build cache -normalizer/lean/.lake/ - -# Local config -.env -.env.local -lith/ - -# Compiled binaries -demo-server - -# Sealed container images (build artifacts) -*.ctp - -# Legacy naming artifacts -lith - -# Historical documentation (Lith IP rename) -Form and Formbase.txt -Lith .txt - -# Node modules (satellite components) -node_modules/ - -# Build artifacts (satellite components) -target/ -.build/ -_build/ -dist/ -lib/ - -# Crash recovery artifacts -ai-cli-crash-capture/ - -# KDE directory metadata -.directory - -# Sync reports -sync_report*.txt -sync_repos_report*.txt - -# Satellite build artifacts -studio/node_modules/ -studio/dist/ -geo/target/ -beam/ebin/ -beam/native_rust/target/ - -# Test package artifacts -tests/*/node_modules/ -tests/*/package.json -tests/*/.deno/ -deps/ -.elixir_ls/ -.cache/ -*.tmp -build/ diff --git a/lithoglyph/.machine_readable/6a2/AGENTIC.a2ml b/lithoglyph/.machine_readable/6a2/AGENTIC.a2ml deleted file mode 100644 index 1699fe4a..00000000 --- a/lithoglyph/.machine_readable/6a2/AGENTIC.a2ml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# AGENTIC.a2ml — AI agent constraints and capabilities -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[agent-permissions] -can-edit-source = true -can-edit-tests = true -can-edit-docs = true -can-edit-config = true -can-create-files = true - -[agent-constraints] -# What AI agents must NOT do: -# - Never use banned language patterns (believe_me, unsafeCoerce, etc.) -# - Never commit secrets or credentials -# - Never use banned languages (TypeScript, Python, Go, etc.) -# - Never place state files in repository root (must be in .machine_readable/) -# - Never use AGPL license (use PMPL-1.0-or-later) - -[maintenance-integrity] -fail-closed = true -require-evidence-per-step = true -allow-silent-skip = false -require-rerun-after-fix = true -release-claim-requires-hard-pass = true - -[automation-hooks] -# on-enter: Read 0-AI-MANIFEST.a2ml, then STATE.a2ml -# on-exit: Update STATE.a2ml with session outcomes -# on-commit: Run just validate-rsr diff --git a/lithoglyph/.machine_readable/6a2/ECOSYSTEM.a2ml b/lithoglyph/.machine_readable/6a2/ECOSYSTEM.a2ml deleted file mode 100644 index d8035f5f..00000000 --- a/lithoglyph/.machine_readable/6a2/ECOSYSTEM.a2ml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# ECOSYSTEM.a2ml — Lithoglyph ecosystem position -[metadata] -version = "1.0" -last-updated = "2026-04-11" - -[project] -name = "Lithoglyph" -purpose = "Narrative-first, reversible, audit-grade database for domains - where provenance, auditability, and human understanding matter - more than raw performance." -role = "database-engine" - -[position-in-ecosystem] -category = "Databases" - -[related-projects] -projects = [ - # No related projects recorded -] diff --git a/lithoglyph/.machine_readable/6a2/META.a2ml b/lithoglyph/.machine_readable/6a2/META.a2ml deleted file mode 100644 index b175db46..00000000 --- a/lithoglyph/.machine_readable/6a2/META.a2ml +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# META.a2ml — Lithoglyph meta-level information -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[project-info] -license = "PMPL-1.0-or-later" -author = "Jonathan D.A. Jewell (hyperpolymath)" - -[architecture-decisions] -decisions = [ - # No ADRs recorded -] - -[development-practices] -versioning = "SemVer" -documentation = "AsciiDoc" -build-tool = "just" - -[maintenance-axes] -scoping-first = true -axis-1 = "must > intend > like" -axis-2 = "corrective > adaptive > perfective" -axis-3 = "systems > compliance > effects" diff --git a/lithoglyph/.machine_readable/6a2/NEUROSYM.a2ml b/lithoglyph/.machine_readable/6a2/NEUROSYM.a2ml deleted file mode 100644 index e1d34c09..00000000 --- a/lithoglyph/.machine_readable/6a2/NEUROSYM.a2ml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# NEUROSYM.a2ml — Neurosymbolic integration metadata -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[hypatia-config] -scan-enabled = true -scan-depth = "standard" # quick | standard | deep -report-format = "logtalk" - -[symbolic-rules] -# Custom symbolic rules for this project -# - { name = "no-unsafe-ffi", pattern = "believe_me|unsafeCoerce", severity = "critical" } - -[neural-config] -# Neural pattern detection settings -# confidence-threshold = 0.85 -# model = "hypatia-v2" diff --git a/lithoglyph/.machine_readable/6a2/PLAYBOOK.a2ml b/lithoglyph/.machine_readable/6a2/PLAYBOOK.a2ml deleted file mode 100644 index 5003fd08..00000000 --- a/lithoglyph/.machine_readable/6a2/PLAYBOOK.a2ml +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# PLAYBOOK.a2ml — Operational playbook -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[deployment] -# method = "gitops" # gitops | manual | ci-triggered -# target = "container" # container | binary | library | wasm - -[incident-response] -# 1. Check .machine_readable/STATE.a2ml for current status -# 2. Review recent commits and CI results -# 3. Run `just validate` to check compliance -# 4. Run `just security` to audit for vulnerabilities - -[release-process] -# 1. Update version in STATE.a2ml, META.a2ml -# 2. Run `just release-preflight` (validate + quality + security + maint-hard-pass) -# 3. Tag and push - -[maintenance-operations] -# Baseline audit: just maint-audit -# Hard release gate: just maint-hard-pass diff --git a/lithoglyph/.machine_readable/6a2/STATE.a2ml b/lithoglyph/.machine_readable/6a2/STATE.a2ml deleted file mode 100644 index b36ea254..00000000 --- a/lithoglyph/.machine_readable/6a2/STATE.a2ml +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# STATE.a2ml — Lithoglyph project state -[metadata] -project = "Lithoglyph" -version = "0.0.7" -last-updated = "2026-03-13" -status = "active" -session = "converted from scheme — 2026-04-11" - -[project-context] -name = "Lithoglyph" -purpose = """C ABI bridge with persistent BlockStorage, WAL commit, 6-phase sync. IP rename complete: fdb_*→lith_*, FQL→GQL, FormBD→Lith.""" -completion-percentage = 65 - -[position] -phase = "ip-rename-complete" # design | implementation | testing | maintenance | archived -maturity = "experimental" # experimental | alpha | beta | production | lts - -[route-to-mvp] -milestones = [ - # No milestones recorded -] - -[blockers-and-issues] -issues = [ - "NAMING-001", -] - -[critical-next-actions] -actions = [ - # No actions recorded -] - -[maintenance-status] -last-run-utc = "2026-03-13T00:00:00Z" -last-result = "unknown" # unknown | pass | warn | fail diff --git a/lithoglyph/.machine_readable/HANDOVER.a2ml b/lithoglyph/.machine_readable/HANDOVER.a2ml deleted file mode 100644 index eb1c44aa..00000000 --- a/lithoglyph/.machine_readable/HANDOVER.a2ml +++ /dev/null @@ -1,404 +0,0 @@ -;; lith.a2ml -;; Lith Unified Handover Artefact (coding-LLM oriented) -;; -;; Conventions: -;; - Treat this file as DATA. Do not evaluate arbitrary code. No macros. -;; - Prefer additive edits. Record decisions in (decisions ...) with rationale. -;; - Every open question has an id + acceptance criteria. - -(define lith - `( - (meta - (scm-format "lith.scm/2") - (data-only true) - (primary-docs - "README.adoc" - "ARCHITECTURE.adoc" - "ROADMAP.adoc" - "PHILOSOPHY.adoc") - (principle "truth-core first, porous edges") - (status - (version "0.0.1") - (stage "conceptual + PoC"))) - - ;; ------------------------------------------------------------ - ;; 0. Identity - ;; ------------------------------------------------------------ - (identity - (name "Lith") - (tagline "The database where the database is part of the story.") - (category "narrative-first, reversible, audit-grade database core") - (stack - (truth-core - (storage "Forth (Form.Blocks)") - (model "Forth (Form.Model)")) - (runtime - (planner "Factor (Form.Runtime)") - (query-language "GQL")) - (interop - (bridge "Zig (Form.Bridge) provides stable C ABI") - (control-plane "Elixir/OTP (optional) for sessions+supervision+cluster edge")))) - - ;; ------------------------------------------------------------ - ;; 1. Mission / non-mission - ;; ------------------------------------------------------------ - (mission - (core-thesis - "Schemas, constraints, migrations, blocks, and journals are narrative artefacts. -The database is part of the story, not an opaque substrate.") - (primary-values - (auditability "> performance") - (meaning "> features") - (reversibility "> throughput") - (agent-understanding "required")) - (target-domains - "investigative journalism" - "governance/compliance" - "agentic ecosystems + multi-repo handover" - "long-term cultural/institutional archives") - (non-goals - "be a drop-in Postgres replacement" - "win microbenchmarks" - "ship full distributed consensus in the first PoC")) - - ;; ------------------------------------------------------------ - ;; 2. Core invariants (non-negotiable) - ;; ------------------------------------------------------------ - (invariants - (truth-ownership - "On-disk truth is owned by the block/journal layer; higher layers do not bypass it.") - (journal-first - "Every mutating operation is journaled before being considered committed.") - (reversibility - "Every committed operation MUST have a defined inverse OR be explicitly marked irreversible-with-story.") - (renderability - "Blocks and journal entries MUST be renderable deterministically into human/agent-readable form.") - (provenance - "All query results can optionally include provenance pointers to journal and blocks.") - (constraints-as-ethics - "Constraints are explainable: rejections must return reasons + pointers + narrative rationale.")) - - ;; ------------------------------------------------------------ - ;; 3. Architectural layers - ;; ------------------------------------------------------------ - (architecture - (layers - (Form.Blocks - (language "Forth") - (purpose "deterministic storage + journal + reversibility primitives") - (must-provide - "fixed-size blocks" - "symbolic headers (stable render)" - "append-only journal" - "crash recovery" - "integrity checks + repair guidance")) - (Form.Model - (language "Forth") - (purpose "multi-model logical layer on top of blocks") - (must-provide - "document collections" - "edge collections" - "schema + constraint metadata" - "migration artefacts") - (note "Model operations are expressed as journaled block operations.")) - (Form.Bridge - (language "Zig") - (purpose "stable ABI boundary + safety governor") - (contract - "Expose a narrow C ABI to runtimes" - "Marshalling only; no business logic duplication" - "Opaque handles + byte buffers + explicit error codes")) - (Form.Runtime - (language "Factor") - (purpose "GQL parse/plan/exec + explain + introspection") - (must-provide - "GQL minimal subset for PoC" - "planner steps introspection" - "constraint explanation surfaces" - "provenance surfaces")) - (Form.ControlPlane - (language "Elixir/OTP (optional)") - (purpose "dependable sessions/supervision/edge clustering") - (rules - "Prefer out-of-process core engine (port) over in-VM native calls" - "Control plane must not redefine truth semantics")) - (Form.Normalizer - (language "Factor + Lean 4 (via GQL-dt)") - (purpose "self-normalizing database: FD discovery, type encoding, proof-carrying schema evolution") - (must-provide - "functional dependency discovery (DFD/TANE/FDHits algorithms)" - "type encoding of FDs in GQL-dt" - "normal form predicates (1NF through BCNF)" - "proposal generation with equivalence proofs" - "narrative templates for normalization decisions" - "DISCOVER DEPENDENCIES command" - "APPLY NORMALIZATION command with rollback") - (spec "spec/self-normalizing.adoc")))) - - ;; ------------------------------------------------------------ - ;; 4. Cross-layer seam checks (must run at stage freezes) - ;; ------------------------------------------------------------ - (seams - (B<->M - "Every Model op maps to a sequence of journaled Block ops." - "Every Model op has an inverse mapping or is explicitly classified.") - (M<->R - "Constraints enforced identically whether invoked via GQL or direct API." - "Introspection must return reason graphs + provenance pointers.") - (B<->R - "Runtime cannot commit without journal-first acknowledgement from Blocks." - "Render tools must work without Factor runtime present.")) - - ;; ------------------------------------------------------------ - ;; 5. Interfaces: Zig C ABI (minimum for PoC) - ;; ------------------------------------------------------------ - (abi - (style "C ABI provided by Zig") - (handles - (db "opaque lith_db*") - (txn "opaque lith_txn*") - (cursor "opaque lith_cursor*")) - (errors - (model "status code + optional error blob") - (rules - "No exceptions across ABI" - "Every error blob is renderable/explainable"))) - (functions - ;; Lifecycle - "lith_db_open(path, opts_bytes, opts_len) -> (db*, status, err_blob)" - "lith_db_close(db*) -> status" - ;; Transactions - "lith_txn_begin(db*, mode) -> (txn*, status, err_blob)" - "lith_txn_commit(txn*) -> (status, err_blob)" - "lith_txn_abort(txn*) -> status" - ;; Apply operations (runtime provides op blob; core returns result+provenance) - "lith_apply(txn*, op_bytes, op_len) -> (result_blob, provenance_blob, status, err_blob)" - ;; Introspection/rendering (must be usable by agents) - "lith_render_block(db*, block_id, render_opts) -> (text_blob, status, err_blob)" - "lith_render_journal(db*, since, render_opts) -> (text_blob, status, err_blob)" - "lith_introspect_schema(db*) -> (schema_blob, status, err_blob)" - "lith_introspect_constraints(db*) -> (constraints_blob, status, err_blob)") - (blob-encodings - (preferred - "Cap'n Proto (if adopted) OR Protobuf (if adopted) for ABI blobs" - "CBOR or MessagePack acceptable for PoC") - (rule - "Regardless of blob encoding, provide deterministic text render for audit."))) - - ;; ------------------------------------------------------------ - ;; 6. On-disk formats (spec-first) - ;; ------------------------------------------------------------ - (formats - (on-disk - (must - "block header: versioned + fixed field layout" - "journal entry: op type + forward payload + inverse payload + provenance ids" - "canonical render: deterministic, stable across implementations") - (deliverables - "spec/blocks.adoc" - "spec/journal.adoc" - "spec/rendering.adoc" - "test-vectors/ (golden bytes + golden renders)")) - (wire - (note "Wire protocol comes after PoC; ABI is enough initially.") - (candidates "HTTP+CBOR" "gRPC" "WebSocket streaming"))) - - ;; ------------------------------------------------------------ - ;; 7. GQL (PoC subset + introspection) - ;; ------------------------------------------------------------ - (gql - (poс-subset - (must - "INSERT document into collection" - "INSERT edge (from,to,type,props)" - "SELECT with simple predicates" - "EXPLAIN (returns plan + reasons)" - "INTROSPECT schema/constraints" - "OPTIONAL provenance output")) - (non-goals - "full SQL equivalence" - "complex joins/aggregations in PoC")) - - ;; ------------------------------------------------------------ - ;; 8. PoC acceptance (definition of done) - ;; ------------------------------------------------------------ - (poc - (acceptance - "single-node db open/close" - "append-only journal with deterministic rendering" - "document+edge insert/select" - "constraint rejection returns explain payload" - "migration artefact recorded + reversible" - "golden test vectors pass" - "seam checks B<->M, M<->R, B<->R pass at freeze")) - - ;; ------------------------------------------------------------ - ;; 9. Work rules for coding LLMs (very explicit) - ;; ------------------------------------------------------------ - (llm-rules - (must - "Keep the truth-core small, readable, test-vector driven." - "Do not duplicate semantics across layers." - "When adding features, add a renderable narrative delta." - "When closing an open question, add a decision record.") - (must-not - "Turn Lith into generic CRUD" - "Hide block/journal meaning behind opaque structures" - "Introduce irreversible operations without explicit classification + story" - "Make runtime capable of bypassing journal-first")) - - ;; ------------------------------------------------------------ - ;; 10. Open questions (structured) - ;; ------------------------------------------------------------ - (open-questions - (q (id "Q-BLOCK-HEADER-001") (area storage) (status resolved) - (text "What is the minimal block header layout (fields + sizes) that supports renderability, integrity, and forward compatibility?") - (acceptance - "fields enumerated + fixed sizes" - "versioning + reserved bits defined" - "canonical rendering rules defined") - (impacts "spec/blocks.adoc" "core-forth/Form.Blocks/*") - (resolved-by "D-BLOCK-HEADER-001")) - (q (id "Q-JOURNAL-ENTRY-001") (area storage) (status resolved) - (text "What is the minimal journal entry schema to guarantee reversibility and provenance pointers?") - (acceptance - "forward+inverse payload representation chosen" - "provenance ids defined" - "crash recovery rules defined") - (impacts "spec/journal.adoc" "core-forth/Form.Blocks/*") - (resolved-by "D-JOURNAL-ENTRY-001")) - (q (id "Q-ABI-BLOBS-001") (area interop) (status resolved) - (text "Choose ABI blob encoding: Cap'n Proto vs Protobuf vs CBOR/MsgPack for PoC.") - (acceptance - "one chosen for PoC" - "deterministic text render defined independent of blob encoding") - (impacts "core-zig/Form.Bridge/*" "core-factor/Form.Runtime/*") - (resolved-by "D-ABI-BLOBS-001")) - (q (id "Q-GQL-POC-001") (area gql) (status resolved) - (text "Define the exact PoC grammar + canonical examples for GQL.") - (acceptance - "grammar documented" - "10 example queries + expected outputs" - "EXPLAIN/INTROSPECT included") - (impacts "spec/gql.adoc" "core-factor/Form.Runtime/*") - (resolved-by "D-GQL-POC-001")) - (q (id "Q-CTRL-PLANE-001") (area control-plane) (status resolved) - (text "Is Elixir/OTP introduced at PoC time (gateway only), or deferred until after the core is stable?") - (acceptance - "decision recorded with rationale" - "if yes: port protocol defined; if no: deferral rationale documented") - (impacts "control-plane/*" "docs/ARCHITECTURE.adoc") - (resolved-by "D-CTRL-PLANE-001")) - ;; Self-Normalizing Database Questions - (q (id "Q-NORM-001") (area normalizer) (status resolved) - (text "Which FD discovery algorithm should be the default: DFD, TANE, or FDHits?") - (acceptance - "benchmark on representative datasets completed" - "accuracy/speed tradeoff documented" - "default chosen with rationale") - (impacts "core-factor/Form.Normalizer/*" "spec/self-normalizing.adoc") - (resolved-by "D-NORM-001")) - (q (id "Q-NORM-002") (area normalizer) (status resolved) - (text "How should approximate FDs (confidence < 1.0) be handled?") - (acceptance - "policy for near-FDs defined" - "data quality implications documented" - "threshold configuration supported") - (impacts "core-factor/Form.Normalizer/*" "spec/self-normalizing.adoc") - (resolved-by "D-NORM-002")) - (q (id "Q-NORM-003") (area normalizer) (status resolved) - (text "Should denormalization be supported with the same rigor as normalization?") - (acceptance - "if yes: DenormalizationStep type defined with proofs" - "performance optimization use cases documented" - "reversibility guarantees specified") - (impacts "spec/self-normalizing.adoc" "core-factor/Form.Normalizer/*") - (resolved-by "D-NORM-003")) - (q (id "Q-NORM-004") (area normalizer) (status resolved) - (text "How to integrate Form.Normalizer with GQL-dt's existing proof system?") - (acceptance - "interface between Lean 4 proofs and Form.Normalizer defined" - "proof verification flow documented" - "bidirectional FFI via Form.Bridge specified") - (impacts "core-zig/Form.Bridge/*" "gqldt/*" "spec/self-normalizing.adoc") - (resolved-by "D-NORM-004")) - (q (id "Q-NORM-005") (area normalizer) (status resolved) - (text "What happens when normalization would break existing queries?") - (acceptance - "query rewriting strategy defined" - "migration period policy documented" - "backward compatibility guarantees specified") - (impacts "core-factor/Form.Runtime/*" "spec/self-normalizing.adoc") - (resolved-by "D-NORM-005"))) - - ;; ------------------------------------------------------------ - ;; 11. Decisions log (append-only) - ;; ------------------------------------------------------------ - (decisions - ;; (d (id "D-...") (date "YYYY-MM-DD") (closes "Q-...") (decision "...") (rationale "...") (impacts "...")) - - (d (id "D-BLOCK-HEADER-001") (date "2026-01-11") (closes "Q-BLOCK-HEADER-001") - (decision "4096-byte blocks with 64-byte fixed header") - (rationale "Matches filesystem/SSD page sizes. Header provides: magic (4), version (2), type (2), block_id (8), sequence (8), timestamps (16), payload_len (4), checksum (4), prev_block (8), flags (4), reserved (4). CRC32C for integrity. Full spec in spec/blocks.adoc.") - (impacts "spec/blocks.adoc")) - - (d (id "D-JOURNAL-ENTRY-001") (date "2026-01-11") (closes "Q-JOURNAL-ENTRY-001") - (decision "48-byte entry header with CBOR payloads for forward/inverse/provenance") - (rationale "Header includes: sequence (8), timestamp (8), op_type (2), flags (2), lengths (12), affected_block (8), checksum (4), entry_len (4). Variable-length CBOR payloads for forward operation, inverse operation, and provenance (actor + rationale required). Crash recovery via journal replay. Full spec in spec/journal.adoc.") - (impacts "spec/journal.adoc")) - - (d (id "D-ABI-BLOBS-001") (date "2026-01-11") (closes "Q-ABI-BLOBS-001") - (decision "CBOR (RFC 8949) with deterministic encoding") - (rationale "Schema-optional, self-describing, wide language support (Zig, Factor, Forth, Lean 4, Elixir). Deterministic encoding via RFC 8949 §4.2 rules. Lith-specific tags 39001-39008 for block refs, doc IDs, provenance, PROMPT scores, proofs. LZ4 compression for large payloads. Full spec in spec/encoding.adoc.") - (impacts "spec/encoding.adoc" "core-zig/Form.Bridge/*")) - - (d (id "D-GQL-POC-001") (date "2026-01-11") (closes "Q-GQL-POC-001") - (decision "GQL PoC grammar with 10 canonical examples") - (rationale "Covers INSERT/SELECT/UPDATE/DELETE for documents and edges. CREATE/DROP for collections. TRAVERSE for edge traversal. EXPLAIN shows plan + rationale. INTROSPECT for schema/constraints/journal. WITH PROVENANCE for audit output. All errors include rationale + suggestions. Full grammar and 10 examples in spec/gql.adoc.") - (impacts "spec/gql.adoc" "core-factor/Form.Runtime/*")) - - (d (id "D-CTRL-PLANE-001") (date "2026-01-12") (closes "Q-CTRL-PLANE-001") - (decision "Defer Elixir/OTP control plane until after core is stable") - (rationale "PoC goal is proving narrative-first, reversible database concept. Control plane doesn't own truth semantics (per architecture). Form.Bridge provides sufficient ABI for external orchestration. Adding Elixir/OTP adds complexity without proving core thesis. Can integrate after Form.Runtime is complete and tested. Port protocol will be defined when needed.") - (impacts "docs/ARCHITECTURE.adoc" "ROADMAP.adoc")) - - (d (id "D-NORM-001") (date "2026-01-12") (closes "Q-NORM-001") - (decision "DFD (Depth-First Discovery) as default FD discovery algorithm") - (rationale "DFD is sample-based, making it practical for large datasets without loading entire relations into memory. More memory-efficient than TANE (no full lattice needed). Already scaffolded in fd-discovery.factor. FDHits is newer (2024) but less battle-tested. For Lith's target use cases (journalism, governance, archives), accuracy matters more than microsecond-level speed. Can add TANE (--algorithm tane) and FDHits (--algorithm fdhits) as alternatives later. DFD's depth-first approach finds minimal FDs efficiently.") - (impacts "core-factor/Form.Normalizer/*" "spec/self-normalizing.adoc")) - - (d (id "D-NORM-002") (date "2026-01-12") (closes "Q-NORM-002") - (decision "Three-tier policy for approximate FDs based on confidence thresholds") - (rationale "Policy: (1) Exact FDs (conf >= 0.99): treat as true FDs, can trigger normalization proposals. (2) Strong approximate FDs (0.95 <= conf < 0.99): report as 'probable FDs', require human confirmation before normalization. (3) Weak approximate FDs (conf < 0.95): report as 'data quality warnings', never trigger normalization. Approximate FDs often indicate data quality issues (typos, legacy data, duplicate records). Surfacing them as warnings supports Lith's audit mission. Requiring confirmation prevents false positives from driving schema changes. Threshold configurable via confidence-threshold setting (default 0.95).") - (impacts "core-factor/Form.Normalizer/*" "spec/self-normalizing.adoc")) - - (d (id "D-NORM-003") (date "2026-01-12") (closes "Q-NORM-003") - (decision "Yes, support denormalization with same rigor as normalization") - (rationale "Define DenormalizationStep type parallel to NormalizationStep in Lean 4. Require performance justification narrative (explaining read optimization goals). Store equivalence proof (same as normalization - join of denormalized is lossless). Journal with explicit 'intentional-denormalization' classification and CBOR tag. Real workloads sometimes need denormalization for read performance. Lith's 'reversibility' invariant means denormalization must be undoable. 'Constraints as ethics' means denormalization must explain trade-offs. Without rigorous denormalization support, users would work around the system.") - (impacts "spec/self-normalizing.adoc" "core-factor/Form.Normalizer/*" "normalizer/lean/FunDep.lean")) - - (d (id "D-NORM-004") (date "2026-01-12") (closes "Q-NORM-004") - (decision "Form.Bridge exports proof verification FFI with external Lean 4 verifiers") - (rationale "Interface: (1) Form.Bridge exports lith_proof_verify(proof_blob, len) -> (valid, err_blob) and lith_proof_register_verifier(type, callback) -> status. (2) Lean 4 proofs compile to standalone C-ABI-compatible verifiers via lake build. (3) Proof references in journal entries use CBOR tag 39006. (4) Flow: Form.Normalizer (Factor) -> Form.Bridge (Zig) -> Lean 4 verifier -> result. This keeps proofs external to truth core (maintains 'truth core doesn't own semantics'). Form.Bridge already provides ABI boundary; proof verification is another FFI call. Lean 4's native compilation to C makes this practical. GQL-dt proofs become verifiable artefacts, not just documentation.") - (impacts "core-zig/Form.Bridge/*" "normalizer/lean/*" "spec/self-normalizing.adoc")) - - (d (id "D-NORM-005") (date "2026-01-12") (closes "Q-NORM-005") - (decision "Three-phase migration with query rewriting: Announce -> Shadow -> Commit") - (rationale "Phase 1 Announce (configurable, default 24h): Normalization proposal journaled; affected queries identified via Form.Runtime query log analysis; applications warned via INTROSPECT warnings. Phase 2 Shadow (configurable, default 7 days): Both old and new schemas exist; queries auto-rewritten to equivalent joins on new schema; compatibility views created automatically; performance metrics collected to validate no regression. Phase 3 Commit: Old schema removed; rewrite rules made permanent; compatibility views removed. Query rewriting rules: SELECT FROM old_table -> SELECT FROM (JOIN new_tables ON split_key). Lith's audience (journalism, governance, archives) needs high availability. Breaking changes must be phased, not abrupt. Shadow phase allows testing before commit. Provenance tracking identifies all affected queries.") - (impacts "core-factor/Form.Runtime/*" "spec/self-normalizing.adoc")) - ) - - ;; ------------------------------------------------------------ - ;; 12. Repo layout (suggested) - ;; ------------------------------------------------------------ - (repo - (dirs - (spec "format specs + rationale (AsciiDoc)") - (core-forth "Form.Blocks + Form.Model (truth core)") - (core-zig "Form.Bridge (ABI + port framing)") - (core-factor "Form.Runtime (GQL + introspection)") - (control-plane "Elixir/OTP gateway (optional)") - (tools "render/inspect/doctor utilities") - (test-vectors "golden bytes + golden renders") - (stories "narrative examples + onboarding/handover artefacts"))) -)) diff --git a/lithoglyph/.machine_readable/ROADMAP.a2ml b/lithoglyph/.machine_readable/ROADMAP.a2ml deleted file mode 100644 index cf7f8ed2..00000000 --- a/lithoglyph/.machine_readable/ROADMAP.a2ml +++ /dev/null @@ -1,248 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith Ecosystem - Unified Roadmap to MVP 1.0.0 -; Media-Type: application/vnd.roadmap+scm -; -; This file is distributed to all Lith ecosystem repos: -; - lith (core database) -; - gql-dt (dependently-typed query language) -; - lithoglyph-studio (GUI) -; - lithoglyph-debugger (recovery tool) -; - lithoglyph-geo (geospatial projection layer) -; - lithoglyph-analytics (OLAP analytics projection layer) - -(unified-roadmap - (metadata - (version "1.1.0") - (created "2026-01-12") - (updated "2026-01-16") - (author "hyperpolymath") - (target "MVP 1.0.0")) - - ;; ============================================================================ - ;; ECOSYSTEM OVERVIEW - ;; ============================================================================ - (ecosystem-summary - (components - (lith - (version "0.0.5") - (completion 80) - (role "Core database engine") - (tech "Forth + Factor + Zig")) - (gql-dt - (version "0.2.0") - (completion 65) - (role "Dependently-typed query language") - (tech "Lean 4 + Zig")) - (lithoglyph-studio - (version "0.1.0") - (completion 45) - (role "Zero-friction GUI") - (tech "ReScript + Tauri 2.0 + Rust")) - (lithoglyph-debugger - (version "0.1.0") - (completion 55) - (role "Proof-carrying recovery tool") - (tech "Lean 4 + Idris 2 + Rust")) - (lithoglyph-geo - (version "0.1.0") - (completion 15) - (role "Geospatial projection layer") - (tech "Rust (rstar, axum, geo)")) - (lithoglyph-analytics - (version "0.1.0") - (completion 15) - (role "OLAP analytics projection layer") - (tech "Julia (DataFrames, Parquet2, Oxygen)"))) - - (architecture - "┌─────────────────────────────────────────────────────────────┐" - "│ Lith Studio (GUI) │" - "│ ↓ generates FQLdt code │" - "├─────────────────────────────────────────────────────────────┤" - "│ FQLdt (Lean 4) │" - "│ ↓ compiles to proof blobs │" - "├─────────────────────────────────────────────────────────────┤" - "│ Form.Bridge (Zig ABI) │" - "│ ↓ calls │" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Core (Forth + Factor) │" - "│ Form.Runtime → Form.Normalizer → Form.Model → Form.Blocks│" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Debugger (alongside) │" - "│ ↓ proves recovery safe │" - "│ Lith + FQLdt │" - "├─────────────────────────────────────────────────────────────┤" - "│ Projection Layers (read from Lith HTTP API) │" - "│ lithoglyph-geo (R-tree spatial) lithoglyph-analytics (OLAP) │" - "└─────────────────────────────────────────────────────────────┘")) - - ;; ============================================================================ - ;; CRITICAL PATH TO MVP 1.0.0 - ;; ============================================================================ - (critical-path - (phase (id "P1") (name "Core Integration") - (duration "weeks 1-6") - (focus "Lith + FQLdt integration") - - (lithoglyph-tasks - (task "Complete M11: HTTP API Server" priority: critical status: complete) - (task "M12: Language bindings (ReScript, PHP)" priority: critical status: next) - (task "M13: CMS integration (WordPress)" priority: high status: pending) - (task "M14: Form.ControlPlane (clustering)" priority: medium status: pending)) - - (gql-dt-tasks - (task "M5: Zig FFI bridge to Form.Bridge" priority: critical status: not-started) - (task "M6: GQL parser (integrate with Lith's EBNF)" priority: high status: not-started) - (task "Proof blob serialization (CBOR RFC 8949)" priority: high status: pending)) - - (checkpoint "FQLdt can compile a query → proof blob → Lith accepts and executes")) - - (phase (id "P2") (name "User-Facing Tools") - (duration "weeks 7-10") - (focus "Studio and Debugger completion") - - (studio-tasks - (task "Verify ReScript/Tauri build pipeline" priority: critical status: pending) - (task "Wire ReScript UI to FQLdt code generation" priority: high status: pending) - (task "Connect to Lith HTTP API" priority: high status: blocked) - (task "Test schema creation → query → results flow" priority: medium status: pending)) - - (debugger-tasks - (task "Wire Idris REPL to PostgreSQL adapter" priority: high status: pending) - (task "Lith adapter: parse real journal files" priority: high status: partial) - (task "Complete Ratatui TUI interface" priority: medium status: in-progress) - (task "Integration: proof verification before recovery" priority: medium status: pending)) - - (projection-layer-tasks - (task "lithoglyph-geo: Integration test with real Lith" priority: high status: pending) - (task "lithoglyph-geo: Docker deployment" priority: medium status: pending) - (task "lithoglyph-analytics: Integration test with real Lith" priority: high status: pending) - (task "lithoglyph-analytics: PROMPT score dashboard endpoints" priority: medium status: pending)) - - (checkpoint "Users can create schemas in Studio, debug with Debugger, query spatial/analytics")) - - (phase (id "P3") (name "Production Hardening") - (duration "weeks 11-12") - (focus "Stability and polish") - - (all-repos - (task "Crash recovery tests" priority: high) - (task "Error handling improvements" priority: high) - (task "Cross-platform testing" priority: medium) - (task "Documentation completion" priority: medium) - (task "Performance optimization" priority: low)) - - (checkpoint "MVP 1.0.0 release ready"))) - - ;; ============================================================================ - ;; DEPENDENCY GRAPH - ;; ============================================================================ - (dependencies - (lithoglyph-m11 - (name "Lith HTTP API Server") - (blocks "Studio M2" "Debugger Lith adapter") - (priority critical)) - - (gql-dt-m5 - (name "FQLdt Zig FFI Bridge") - (blocks "Studio M3" "Real type checking") - (depends-on "Lith Form.Bridge") - (priority critical)) - - (gql-dt-m6 - (name "FQLdt GQL Parser") - (blocks "Full FQLdt compilation") - (depends-on "gql-dt-m5") - (priority high)) - - (studio-m1 - (name "Studio Build Pipeline") - (blocks "All Studio features") - (priority critical)) - - (debugger-repl-db - (name "Debugger REPL Database Connection") - (blocks "Real debugging") - (priority high)) - - (lithoglyph-geo-integration - (name "lithoglyph-geo Lith Integration") - (depends-on "lithoglyph-m11") - (blocks "Spatial queries in Studio") - (priority medium)) - - (lithoglyph-analytics-integration - (name "lithoglyph-analytics Lith Integration") - (depends-on "lithoglyph-m11") - (blocks "Analytics dashboards in Studio") - (priority medium))) - - ;; ============================================================================ - ;; UNRESOLVED DECISIONS - ;; ============================================================================ - (decisions-needed - (decision (id "DECISION-002") - (title "FQLdt parser approach") - (repo "gql-dt") - (options - "Hand-rolled parser (simple, no deps)" - "Lean 4 Parsec (built-in)" - "Integrate with Lith's Factor-based GQL parser") - (recommendation "Integrate - reuse Lith's EBNF grammar via FFI") - (impact "Affects M6 implementation")) - - (decision (id "DECISION-003") - (title "Lith integration strategy for FQLdt") - (repo "gql-dt") - (options - "Mock Forth core for MVP" - "Real Form.Bridge integration") - (recommendation "Real integration - M11 HTTP API makes this feasible") - (impact "Determines MVP scope"))) - - ;; ============================================================================ - ;; POST-MVP ROADMAP - ;; ============================================================================ - (post-mvp - (release (version "1.1.0") (name "Normalization & Migration") - (features - "Form.Normalizer full integration (FD discovery → decomposition)" - "Three-phase migration workflow (Announce/Shadow/Commit)" - "Studio: visual normalization wizard" - "Debugger: migration rollback proofs" - "lithoglyph-geo: Polygon and region queries" - "lithoglyph-analytics: Time-series dashboards in Studio")) - - (release (version "1.2.0") (name "Multi-Database Support") - (features - "Debugger: SQLite adapter completion" - "Lith: clustering/replication (Form.ControlPlane begins)" - "Studio: connection manager for multiple DBs")) - - (release (version "2.0.0") (name "Agentic Ecosystem") - (features - "Form.ControlPlane (Elixir/OTP) for distributed coordination" - "Agent handover protocols" - "Long-term archive format standardization" - "Multi-user collaboration in Studio"))) - - ;; ============================================================================ - ;; SUCCESS METRICS - ;; ============================================================================ - (success-metrics - (mvp-criteria - "User can create a schema in Studio with visual builder" - "Schema generates valid FQLdt with type checking" - "User can insert data with provenance tracking" - "User can query data and see results" - "Debugger can analyze schema and propose fixes" - "All operations have proof-carrying verification") - - (quality-gates - "All ReScript code compiles without warnings" - "All Rust code passes Clippy lints" - "All Lean 4 code builds with lake" - "All Julia code passes tests" - "Cross-platform builds succeed (Mac/Windows/Linux)" - "Integration tests pass end-to-end" - "Projection layers can sync from Lith HTTP API"))) diff --git a/lithoglyph/.meta/REQUIRED-FILES.md b/lithoglyph/.meta/REQUIRED-FILES.md deleted file mode 100644 index b06e2061..00000000 --- a/lithoglyph/.meta/REQUIRED-FILES.md +++ /dev/null @@ -1,53 +0,0 @@ -# Required Repository Files - -The following files **MUST** be present and kept up-to-date in every repository: - -## Mandatory Dotfiles - -| File | Purpose | -|------|---------| -| `.gitignore` | Exclude build artifacts, secrets, and temp files | -| `.gitattributes` | Enforce LF line endings and diff settings | -| `.editorconfig` | Consistent editor settings across IDEs | -| `.tool-versions` | asdf version pinning for reproducible builds | - -## Mandatory SCM Files - -| File | Purpose | -|------|---------| -| `META.scm` | Architecture decisions, development practices | -| `STATE.scm` | Project state, phase, milestones | -| `ECOSYSTEM.scm` | Ecosystem positioning, related projects | -| `PLAYBOOK.scm` | Executable plans, procedures | -| `AGENTIC.scm` | AI agent operational gating | -| `NEUROSYM.scm` | Symbolic semantics, proof obligations | - -## Build System - -| File | Purpose | -|------|---------| -| `justfile` | Task runner (replaces Makefile) | -| `Mustfile` | Deployment state contract | - -**IMPORTANT**: Makefiles are FORBIDDEN. Use `just` for all tasks. - -## Validation - -These files are checked by: -- CI workflow validation -- Pre-commit hooks (when configured) -- Repository standardization scripts - -## Updates - -When updating these files: -1. Use templates from `rsr-template-repo` as reference -2. Ensure SPDX license header is present -3. Test changes locally before pushing -4. Keep language-specific sections relevant to the repo - -## See Also - -- [RSR (Rhodium Standard Repositories)](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [Mustfile Specification](https://github.com/hyperpolymath/mustfile) -- [SCM Format Family](https://github.com/hyperpolymath/meta-scm) diff --git a/lithoglyph/.tool-versions b/lithoglyph/.tool-versions deleted file mode 100644 index 332f9613..00000000 --- a/lithoglyph/.tool-versions +++ /dev/null @@ -1,2 +0,0 @@ -idris2 0.7.0 -just 1.36.0 diff --git a/lithoglyph/.well-known/groove/manifest.json b/lithoglyph/.well-known/groove/manifest.json deleted file mode 100644 index 581ab4f3..00000000 --- a/lithoglyph/.well-known/groove/manifest.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "groove_version": "1", - "service_id": "lithoglyph", - "service_version": "0.1.0", - "mode": "active", - "capabilities": { - "narrative-storage": { - "type": "data-storage", - "description": "Narrative-first, reversible, audit-grade database core for journalism, storytelling, and media workflows — schemas and migrations as permanent narrative artefacts carved in Forth", - "protocol": "http", - "endpoint": "/api", - "requires_auth": false, - "panel_compatible": true - }, - "graphql-query": { - "type": "data-storage", - "description": "GraphQL interface for Lith queries, mutations, and subscriptions (SDL-based)", - "protocol": "http", - "endpoint": "/graphql", - "requires_auth": false, - "panel_compatible": true - }, - "grpc-interface": { - "type": "data-storage", - "description": "gRPC interface for Lith operations over HTTP/2 using Protocol Buffers", - "protocol": "http", - "endpoint": "/grpc", - "requires_auth": false, - "panel_compatible": false - } - }, - "consumes": [], - "endpoints": { - "health": "/health" - }, - "health": "/health", - "ports": { - "http": 8080 - }, - "applicability": ["individual", "team"] -} diff --git a/lithoglyph/.well-known/security.txt b/lithoglyph/.well-known/security.txt deleted file mode 100644 index b4230ba6..00000000 --- a/lithoglyph/.well-known/security.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Security contact for Lith -# See https://securitytxt.org/ - -Contact: https://github.com/hyperpolymath/lithoglyph/security/advisories/new -Preferred-Languages: en -Canonical: https://github.com/hyperpolymath/lithoglyph/.well-known/security.txt -Policy: https://github.com/hyperpolymath/lithoglyph/blob/main/SECURITY.md diff --git a/lithoglyph/0-AI-MANIFEST.a2ml b/lithoglyph/0-AI-MANIFEST.a2ml deleted file mode 100644 index d1ba0f83..00000000 --- a/lithoglyph/0-AI-MANIFEST.a2ml +++ /dev/null @@ -1,149 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -; -; 0-AI-MANIFEST.a2ml — Universal AI entry point for Lithoglyph -; Media-Type: application/a2ml - -(manifest - (identity - (name "Lithoglyph") - (full-name "Lithoglyph: Narrative-First, Reversible, Audit-Grade Database") - (former-name "Lith") - (version "0.0.7") - (repo "https://github.com/hyperpolymath/lithoglyph") - (license "PMPL-1.0-or-later") - (author "Jonathan D.A. Jewell ") - (monorepo-parent "nextgen-databases")) - - (purpose - "Narrative-first, reversible, audit-grade database core. Every mutation - is a story event with full provenance. Block-based storage with Forth - kernel, Zig bridge providing C ABI, Idris2 dependent-type ABI proofs, - Factor runtime with GQL query language, and Lean 4 normalization proofs.") - - (canonical-locations - (ai-instructions ".claude/CLAUDE.md") - (state ".machine_readable/STATE.scm") - (meta ".machine_readable/META.scm") - (ecosystem ".machine_readable/ECOSYSTEM.scm") - (agentic ".machine_readable/AGENTIC.scm") - (neurosym ".machine_readable/NEUROSYM.scm") - (playbook ".machine_readable/PLAYBOOK.scm") - (legacy-handover ".machine_readable/HANDOVER.scm") - (roadmap ".machine_readable/ROADMAP.scm") - (topology "TOPOLOGY.md") - (build "justfile") - (container-build "Containerfile") - (container-deploy "selur-compose.yml") - (honest-status "HONEST-STATUS-2026-02-13.md") - (security ".well-known/security.txt")) - - (tech-stack - (storage-layer "Forth" "Block storage kernel — 17 passing tests") - (bridge-layer "Zig" "C ABI bridge — 19 functions, WAL commit") - (abi-layer "Idris2" "Dependent-type ABI proofs — zero believe_me") - (runtime-layer "Factor" "GQL parser, planner, executor") - (normalizer-layer "Lean 4" "Normalization proofs — 52 passing") - (beam-layer "Zig + Rust" "BEAM NIFs for Elixir/Erlang") - (control-plane "Elixir/OTP" "Clustering, supervision (planned)") - (query-language "GQL-DT" "Dependently-typed, Lean 4 verified") - (http-layer "Elixir" "lith-http — Phoenix-based API") - (config "Nickel") - (container-runtime "Podman") - (base-image "cgr.dev/chainguard/wolfi-base:latest")) - - (architecture - (core-forth - (description "Forth block storage kernel") - (location "core-forth/") - (status "complete — 17/17 tests pass")) - (core-zig - (description "C ABI bridge with WAL, block allocator, compaction") - (location "core-zig/") - (status "complete — build + test pass")) - (ffi-zig - (description "FFI delegation layer — delegates to core-zig") - (location "ffi/zig/") - (status "complete — build + test pass")) - (idris2-abi - (description "Dependent-type ABI definitions with formal proofs") - (location "src/Lith/") - (status "complete — 3 files type-check clean")) - (core-factor - (description "GQL runtime — GQL parser, planner, executor") - (location "core-factor/") - (status "complete — builds")) - (normalizer - (description "Lean 4 normalization with FD discovery") - (location "normalizer/") - (status "complete — 52 proofs pass")) - (beam-zig-nif - (description "Zig-based BEAM NIF") - (location "beam/native/") - (status "builds — real FFI calls")) - (beam-rust-nif - (description "Rust-based BEAM NIF via Rustler 0.35") - (location "beam/native_rust/") - (status "builds — 0 warnings")) - (gql-dt - (description "Dependently-typed Glyph Query Language") - (location "gql-dt/") - (status "claims 100% — needs audit")) - (glyphbase - (description "Graph storage engine") - (location "glyphbase/") - (status "claims complete — needs audit")) - (lith-http - (description "Elixir/Phoenix HTTP API layer") - (location "lith-http/") - (status "M15 complete — production builds")) - (studio - (description "Tauri desktop GUI") - (location "studio/") - (status "11 TODO commands — returns mock data")) - (api - (description "Zig HTTP/gRPC API layer") - (location "api/") - (status "BROKEN — 83 call sites need Zig 0.15.2 migration"))) - - (subproject-manifests - (gql-dt "gql-dt/0-AI-MANIFEST.a2ml") - (glyphbase "glyphbase/0-AI-MANIFEST.a2ml") - (lith-http "lith-http/0-AI-MANIFEST.a2ml")) - - (critical-invariants - (rule "SCM files ONLY in .machine_readable/ — never root") - (rule "Zero believe_me in Idris2 ABI — hard invariant") - (rule "All Zig unsafe casts must have // SAFETY: comments") - (rule "core-zig/src/bridge.zig is the WORKING implementation — ffi/zig/ delegates to it") - (rule "No SQL injection — parameterized query builders only") - (rule "Container runtime is Podman — never Docker") - (rule "Container images MUST use Chainguard base (cgr.dev)") - (rule "IP rename pending: Lith/Form → Litho/Lithoglyph before public release")) - - (container-ecosystem - (selur "selur-compose.yml — deployment orchestration") - (svalinn "TLS gateway with policy enforcement") - (vordr "Runtime verification and formal proof checking") - (cerro-torre "Image signing with ML-DSA-87 post-quantum crypto") - (rokur "Secret rotation with argon2id")) - - (related-projects - (verisimdb "Octad database sibling — shares GQL patterns") - (quandledb "Knot-theoretic database sibling") - (nqc "Normal-form Query Compiler") - (proven "Formally verified safety library") - (hypatia "Neurosymbolic CI/CD scanner") - (gitbot-fleet "Bot orchestration"))) - -## Taxonomy Index - -- `spec/grammar.ebnf` — @taxonomy: spec/grammar -- `spec/README.adoc` — @taxonomy: spec/index -- `verification/README.adoc` — @taxonomy: verification/index -- `gql-dt/vscode-extension/syntaxes/gql-dt.tmLanguage.json` — @taxonomy: editors/textmate-grammar - -### New RSR Standard Directories - -- `spec/` — Canonical specification files -- `verification/` — Unified verification gateway (symlinks to proofs, tests, conformance, benchmarks, fuzzing) diff --git a/lithoglyph/ABI-FFI-README.md b/lithoglyph/ABI-FFI-README.md deleted file mode 100644 index e6a32bbf..00000000 --- a/lithoglyph/ABI-FFI-README.md +++ /dev/null @@ -1,385 +0,0 @@ -{{~ Aditionally delete this line and fill out the template below ~}} - -# {{PROJECT}} ABI/FFI Documentation - -## Overview - -This library follows the **Hyperpolymath RSR Standard** for ABI and FFI design: - -- **ABI (Application Binary Interface)** defined in **Idris2** with formal proofs -- **FFI (Foreign Function Interface)** implemented in **Zig** for C compatibility -- **Generated C headers** bridge Idris2 ABI to Zig FFI -- **Any language** can call through standard C ABI - -## Architecture - -``` -┌─────────────────────────────────────────────┐ -│ ABI Definitions (Idris2) │ -│ src/abi/ │ -│ - Types.idr (Type definitions) │ -│ - Layout.idr (Memory layout proofs) │ -│ - Foreign.idr (FFI declarations) │ -└─────────────────┬───────────────────────────┘ - │ - │ generates (at compile time) - ▼ -┌─────────────────────────────────────────────┐ -│ C Headers (auto-generated) │ -│ generated/abi/{{project}}.h │ -└─────────────────┬───────────────────────────┘ - │ - │ imported by - ▼ -┌─────────────────────────────────────────────┐ -│ FFI Implementation (Zig) │ -│ ffi/zig/src/main.zig │ -│ - Implements C-compatible functions │ -│ - Zero-cost abstractions │ -│ - Memory-safe by default │ -└─────────────────┬───────────────────────────┘ - │ - │ compiled to lib{{project}}.so/.a - ▼ -┌─────────────────────────────────────────────┐ -│ Any Language via C ABI │ -│ - Rust, ReScript, Julia, Python, etc. │ -└─────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -{{project}}/ -├── src/ -│ ├── abi/ # ABI definitions (Idris2) -│ │ ├── Types.idr # Core type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI function declarations -│ └── lib/ # Core library (any language) -│ -├── ffi/ -│ └── zig/ # FFI implementation (Zig) -│ ├── build.zig # Build configuration -│ ├── build.zig.zon # Dependencies -│ ├── src/ -│ │ └── main.zig # C-compatible FFI implementation -│ ├── test/ -│ │ └── integration_test.zig -│ └── include/ -│ └── {{project}}.h # C header (optional, can be generated) -│ -├── generated/ # Auto-generated files -│ └── abi/ -│ └── {{project}}.h # Generated from Idris2 ABI -│ -└── bindings/ # Language-specific wrappers (optional) - ├── rust/ - ├── rescript/ - └── julia/ -``` - -## Why Idris2 for ABI? - -### 1. **Formal Verification** - -Idris2's dependent types allow proving properties about the ABI at compile-time: - -```idris --- Prove struct size is correct -public export -exampleStructSize : HasSize ExampleStruct 16 - --- Prove field alignment is correct -public export -fieldAligned : Divides 8 (offsetOf ExampleStruct.field) - --- Prove ABI is platform-compatible -public export -abiCompatible : Compatible (ABI 1) (ABI 2) -``` - -### 2. **Type Safety** - -Encode invariants that C/Zig cannot express: - -```idris --- Non-null pointer guaranteed at type level -data Handle : Type where - MkHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> Handle - --- Array with length proof -data Buffer : (n : Nat) -> Type where - MkBuffer : Vect n Byte -> Buffer n -``` - -### 3. **Platform Abstraction** - -Platform-specific types with compile-time selection: - -```idris -CInt : Platform -> Type -CInt Linux = Bits32 -CInt Windows = Bits32 - -CSize : Platform -> Type -CSize Linux = Bits64 -CSize Windows = Bits64 -``` - -### 4. **Safe Evolution** - -Prove that new ABI versions are backward-compatible: - -```idris --- Compiler enforces compatibility -abiUpgrade : ABI 1 -> ABI 2 -abiUpgrade old = MkABI2 { - -- Must preserve all v1 fields - v1_compat = old, - -- Can add new fields - new_features = defaults -} -``` - -## Why Zig for FFI? - -### 1. **C ABI Compatibility** - -Zig exports C-compatible functions naturally: - -```zig -export fn library_function(param: i32) i32 { - return param * 2; -} -``` - -### 2. **Memory Safety** - -Compile-time safety without runtime overhead: - -```zig -// Null check enforced at compile time -const handle = init() orelse return error.InitFailed; -defer free(handle); -``` - -### 3. **Cross-Compilation** - -Built-in cross-compilation to any platform: - -```bash -zig build -Dtarget=x86_64-linux -zig build -Dtarget=aarch64-macos -zig build -Dtarget=x86_64-windows -``` - -### 4. **Zero Dependencies** - -No runtime, no libc required (unless explicitly needed): - -```zig -// Minimal binary size -pub const lib = @import("std"); -// Only includes what you use -``` - -## Building - -### Build FFI Library - -```bash -cd ffi/zig -zig build # Build debug -zig build -Doptimize=ReleaseFast # Build optimized -zig build test # Run tests -``` - -### Generate C Header from Idris2 ABI - -```bash -cd src/abi -idris2 --cg c-header Types.idr -o ../../generated/abi/{{project}}.h -``` - -### Cross-Compile - -```bash -cd ffi/zig - -# Linux x86_64 -zig build -Dtarget=x86_64-linux - -# macOS ARM64 -zig build -Dtarget=aarch64-macos - -# Windows x86_64 -zig build -Dtarget=x86_64-windows -``` - -## Usage - -### From C - -```c -#include "{{project}}.h" - -int main() { - void* handle = {{project}}_init(); - if (!handle) return 1; - - int result = {{project}}_process(handle, 42); - if (result != 0) { - const char* err = {{project}}_last_error(); - fprintf(stderr, "Error: %s\n", err); - } - - {{project}}_free(handle); - return 0; -} -``` - -Compile with: -```bash -gcc -o example example.c -l{{project}} -L./zig-out/lib -``` - -### From Idris2 - -```idris -import {{PROJECT}}.ABI.Foreign - -main : IO () -main = do - Just handle <- init - | Nothing => putStrLn "Failed to initialize" - - Right result <- process handle 42 - | Left err => putStrLn $ "Error: " ++ errorDescription err - - free handle - putStrLn "Success" -``` - -### From Rust - -```rust -#[link(name = "{{project}}")] -extern "C" { - fn {{project}}_init() -> *mut std::ffi::c_void; - fn {{project}}_free(handle: *mut std::ffi::c_void); - fn {{project}}_process(handle: *mut std::ffi::c_void, input: u32) -> i32; -} - -fn main() { - unsafe { - let handle = {{project}}_init(); - assert!(!handle.is_null()); - - let result = {{project}}_process(handle, 42); - assert_eq!(result, 0); - - {{project}}_free(handle); - } -} -``` - -### From Julia - -```julia -const lib{{project}} = "lib{{project}}" - -function init() - handle = ccall((:{{project}}_init, lib{{project}}), Ptr{Cvoid}, ()) - handle == C_NULL && error("Failed to initialize") - handle -end - -function process(handle, input) - result = ccall((:{{project}}_process, lib{{project}}), Cint, (Ptr{Cvoid}, UInt32), handle, input) - result -end - -function cleanup(handle) - ccall((:{{project}}_free, lib{{project}}), Cvoid, (Ptr{Cvoid},), handle) -end - -# Usage -handle = init() -try - result = process(handle, 42) - println("Result: $result") -finally - cleanup(handle) -end -``` - -## Testing - -### Unit Tests (Zig) - -```bash -cd ffi/zig -zig build test -``` - -### Integration Tests - -```bash -cd ffi/zig -zig build test-integration -``` - -### ABI Verification (Idris2) - -```idris --- Compile-time verification -%runElab verifyABI - --- Runtime checks -main : IO () -main = do - verifyLayoutsCorrect - verifyAlignmentsCorrect - putStrLn "ABI verification passed" -``` - -## Contributing - -When modifying the ABI/FFI: - -1. **Update ABI first** (`src/abi/*.idr`) - - Modify type definitions - - Update proofs - - Ensure backward compatibility - -2. **Generate C header** - ```bash - idris2 --cg c-header src/abi/Types.idr -o generated/abi/{{project}}.h - ``` - -3. **Update FFI implementation** (`ffi/zig/src/main.zig`) - - Implement new functions - - Match ABI types exactly - -4. **Add tests** - - Unit tests in Zig - - Integration tests - - ABI verification tests - -5. **Update documentation** - - Function signatures - - Usage examples - - Migration guide (if breaking changes) - -## License - -PMPL-1.0-or-later - -## See Also - -- [Idris2 Documentation](https://idris2.readthedocs.io) -- [Zig Documentation](https://ziglang.org/documentation/master/) -- [Rhodium Standard Repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [FFI Migration Guide](../ffi-migration-guide.md) -- [ABI Migration Guide](../abi-migration-guide.md) diff --git a/lithoglyph/ARCHITECTURE.adoc b/lithoglyph/ARCHITECTURE.adoc deleted file mode 100644 index 2d0a27de..00000000 --- a/lithoglyph/ARCHITECTURE.adoc +++ /dev/null @@ -1,277 +0,0 @@ -= Lith Architecture -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font - -[.lead] -Technical deep-dive into Lith's layered architecture. - -toc::[] - -== Overview - -Lith is structured as a stack of layers, each with a specific responsibility and language choice. - -[source,text] ----- -┌─────────────────────────────────────────────────────────────┐ -│ User Applications │ -├─────────────────────────────────────────────────────────────┤ -│ Form.ControlPlane (Elixir/OTP) │ -│ ├── Session management │ -│ ├── Cluster coordination │ -│ └── Port communication with core │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Runtime (Factor) │ -│ ├── GQL parser │ -│ ├── Query planner │ -│ ├── Executor │ -│ └── Introspection surfaces │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Bridge (Zig) │ -│ ├── Zig ABI surface (no C dependency) │ -│ ├── Safety governor │ -│ └── Marshalling │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Model (Forth) │ -│ ├── Document collections │ -│ ├── Edge collections │ -│ ├── Schema metadata │ -│ └── Migration artefacts │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Blocks (Forth) │ -│ ├── Fixed-size blocks │ -│ ├── Append-only journal │ -│ ├── Crash recovery │ -│ └── Integrity checks │ -├─────────────────────────────────────────────────────────────┤ -│ File System │ -└─────────────────────────────────────────────────────────────┘ ----- - -== Layer Details - -=== Form.Blocks (Truth Core - Storage) - -The lowest layer, responsible for durable storage. - -==== Responsibilities - -* **Block Management**: Fixed-size blocks with symbolic headers -* **Journal**: Append-only log of all mutations -* **Recovery**: Crash recovery from journal -* **Integrity**: Checksums and repair guidance - -==== Key Invariants - -* Journal entries are written before block mutations -* All blocks have deterministic text rendering -* No hidden state or implicit behavior - -==== Data Structures - -[source,text] ----- -Block: -┌─────────────────────────────────────────┐ -│ Header (versioned, fixed layout) │ -├─────────────────────────────────────────┤ -│ Payload (block-type specific) │ -└─────────────────────────────────────────┘ - -Journal Entry: -┌─────────────────────────────────────────┐ -│ Sequence number │ -├─────────────────────────────────────────┤ -│ Operation type │ -├─────────────────────────────────────────┤ -│ Forward payload (what was done) │ -├─────────────────────────────────────────┤ -│ Inverse payload (how to undo) │ -├─────────────────────────────────────────┤ -│ Provenance IDs │ -└─────────────────────────────────────────┘ ----- - -=== Form.Model (Truth Core - Logical) - -Multi-model logical layer built on blocks. - -==== Responsibilities - -* **Documents**: JSON-like document storage -* **Edges**: Graph relationships between documents -* **Schemas**: Type and constraint definitions -* **Migrations**: Schema evolution artefacts - -==== Key Invariants - -* All operations translate to journaled block operations -* Every operation has a defined inverse -* Constraints produce explainable rejections - -=== Form.Bridge (Interop) - -Stable ABI boundary between core and runtimes. Written entirely in Zig with no C dependency. - -==== Responsibilities - -* **Zig ABI**: Platform-independent interface via Zig's stable ABI -* **Safety Governor**: Validation at boundary -* **Marshalling**: Data format translation - -==== API Design - -[source,zig] ----- -/// All functions return status + optional error blob -pub const LithStatus = struct { - code: i32, - error_blob: ?[*]const u8, - error_len: usize, -}; - -/// Handles are opaque pointers -pub const LithDb = opaque {}; -pub const LithTxn = opaque {}; -pub const LithCursor = opaque {}; - -/// Example: Open database -pub export fn lith_open(path: [*:0]const u8, path_len: usize) callconv(.C) LithStatus { - // Zig provides C-compatible calling convention for FFI - // without requiring C headers or toolchain -} ----- - -NOTE: Zig's `callconv(.C)` provides C-compatible calling convention for interop with Factor, Elixir ports, and other runtimes, but the implementation is pure Zig with no C headers or libc dependency. - -==== Bidirectional FFI - -Form.Bridge supports bidirectional communication—runtimes can call into the core, and the core can call back into runtimes for constraint checking, provenance validation, and proof verification. - -[source,zig] ----- -/// Forward direction: Runtime → Core -/// Used by GQL (Factor), FQLdt (Lean 4), Elixir control plane -pub export fn lith_insert( - db: *LithDb, - collection: [*:0]const u8, - document: [*]const u8, - doc_len: usize, -) callconv(.C) LithStatus; - -/// Reverse direction: Core → Runtime -/// For constraint checking, proof verification, provenance callbacks -pub export fn lith_register_constraint_checker( - db: *LithDb, - checker: *const fn (doc: [*]const u8, len: usize) callconv(.C) bool, -) callconv(.C) LithStatus; - -pub export fn lith_register_proof_verifier( - db: *LithDb, - verifier: *const fn (proof: [*]const u8, len: usize) callconv(.C) bool, -) callconv(.C) LithStatus; ----- - -This enables: - -* **FQLdt integration**: Lean 4 can register proof verifiers that the core calls before committing -* **Custom constraints**: Runtimes can implement domain-specific validators -* **Provenance hooks**: External systems can be notified of data lineage events - -=== Form.Runtime (Query Layer) - -GQL parser, planner, and executor. - -==== Responsibilities - -* **Parser**: GQL syntax → AST -* **Planner**: AST → execution plan -* **Executor**: Plan → results -* **Introspection**: Schema and constraint queries - -==== GQL Design - -GQL is not SQL. Key differences: - -|=== -| SQL | GQL - -| `SELECT * FROM t` | `SELECT FROM collection` -| Implicit joins | Explicit edge traversal -| `EXPLAIN` shows plan | `EXPLAIN` shows plan + rationale -| No provenance | Optional provenance output -|=== - -=== Form.ControlPlane (Optional) - -Session and cluster management. - -==== Responsibilities - -* **Sessions**: User session lifecycle -* **Supervision**: OTP supervisors for reliability -* **Clustering**: Multi-node coordination - -==== Integration - -Control plane communicates with core via Erlang ports: - -[source,elixir] ----- -# Simplified example -defmodule Lith.Core do - use GenServer - - def init(_) do - port = Port.open({:spawn, "lithoglyph-core"}, [:binary]) - {:ok, %{port: port}} - end -end ----- - -== Seam Checks - -Cross-layer invariants verified at stage freezes: - -=== B↔M (Blocks ↔ Model) - -* Every Model op maps to journaled Block ops -* Every Model op has inverse mapping or classification - -=== M↔R (Model ↔ Runtime) - -* Constraints enforced identically via GQL or direct API -* Introspection returns reason graphs + provenance - -=== B↔R (Blocks ↔ Runtime) - -* Runtime cannot commit without journal-first acknowledgement -* Render tools work without Factor runtime - -== Blob Encoding - -For PoC, we use CBOR for ABI blobs: - -* Compact binary format -* Schema-optional -* Wide language support - -Regardless of encoding, all blobs have deterministic text rendering for audit purposes. - -== Concurrency Model - -* **Single-writer**: One transaction modifies at a time -* **Multi-reader**: Concurrent read transactions -* **MVCC**: (future) Multi-version concurrency control - -== Failure Modes - -|=== -| Failure | Recovery - -| Process crash during write | Replay journal from last checkpoint -| Corruption detected | Integrity check → repair guidance -| Constraint violation | Explainable rejection, no state change -|=== diff --git a/lithoglyph/CHANGELOG.md b/lithoglyph/CHANGELOG.md deleted file mode 100644 index 3d1ac163..00000000 --- a/lithoglyph/CHANGELOG.md +++ /dev/null @@ -1,989 +0,0 @@ -# Changelog - -All notable changes to Lith will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - ---- - -## [2.0.0] - 2026-01-12 - -**Lith 2.0.0 - Feature-Complete Major Release** - -This release marks a major milestone: all 18 milestones complete, representing the feature-complete vision of Lith as a narrative-first, self-normalizing database. - -### Highlights - -- **18 Milestones Complete** - From M1 (Specification) through M18 (Advanced Analytics) -- **Distributed Mode** - Full cluster coordination, Raft consensus, sharding, and replication -- **Advanced Analytics** - Statistical aggregations, time series analysis, window functions -- **Production Hardened** - Health checks, graceful shutdown, configuration validation -- **Multi-Protocol API** - REST, gRPC, GraphQL, WebSocket -- **CMS Integrations** - Strapi, Directus, Ghost, Payload -- **Client Libraries** - ReScript, PHP with full SDK generator - -### What's New Since 1.0.0 - -#### Distributed Computing (v1.1.0) -- Cluster coordination with node discovery and membership -- Raft consensus for leader election and log replication -- Consistent hashing with virtual nodes for data sharding -- Configurable consistency levels (One, Quorum, All) - -#### Advanced Analytics (v1.2.0) -- Statistical aggregations (sum, avg, stddev, percentile, etc.) -- Time series analysis with trend detection and anomaly detection -- SQL-style window functions (Rank, Lag, Lead, CumulativeSum) -- Visualization exports (CSV, JSON, Chart.js, Vega-Lite, D3) - -### Complete Feature Set - -| Category | Features | -|----------|----------| -| **Storage** | 4KiB blocks, CRC32C integrity, append-only journal | -| **Query** | GQL language, query planner, EXPLAIN modes | -| **Normalization** | DFD discovery, 1NF-BCNF analysis, three-phase migration | -| **API** | REST, gRPC, GraphQL, WebSocket subscriptions | -| **Distributed** | Cluster, Raft consensus, sharding, replication | -| **Analytics** | Aggregations, time series, window functions, exports | -| **Performance** | Query cache, connection pool, batch operations, metrics | -| **Stability** | Config validation, health checks, graceful shutdown | -| **Clients** | ReScript, PHP with SDK generator | -| **CMS** | Strapi, Directus, Ghost, Payload integrations | -| **Testing** | Property-based, fuzz, integration, E2E | - -### Breaking Changes - -None - v2.0.0 maintains backward compatibility with the v1.x series. - -### Thank You - -Lith 2.0.0 represents the complete realization of the narrative-first database vision. Every schema change, every constraint, every migration is now a story that can be told, verified, and understood. - ---- - -## [1.2.0] - 2026-01-12 - -Advanced Analytics milestone: **M18 Complete** - -This release adds comprehensive analytics capabilities to Lith. - -### Added - -#### Aggregations (`analytics/src/Lith_Analytics_Aggregations.res`) -- Statistical aggregations: Count, Sum, Avg, Min, Max -- Advanced: Median, Stddev, Variance, Percentile -- CountDistinct for unique value counting -- Group by with multiple aggregations -- Numeric value extraction from JSON - -#### Time Series (`analytics/src/Lith_Analytics_TimeSeries.res`) -- Time bucketing (Second to Year granularity) -- Trend analysis with linear regression -- Moving average calculation -- Anomaly detection using standard deviation -- R-squared and slope metrics - -#### Window Functions (`analytics/src/Lith_Analytics_Window.res`) -- Ranking: RowNumber, Rank, DenseRank, PercentRank, Ntile -- Navigation: Lag, Lead, FirstValue, LastValue, NthValue -- Running calculations: CumulativeSum, RunningAvg -- Partition by and order by support -- Configurable frame boundaries - -#### Visualization Export (`analytics/src/Lith_Analytics_Export.res`) -- CSV export with configurable options -- JSON export for APIs -- Chart.js configuration export -- Vega-Lite specification export -- D3-compatible data format -- Table data format (columns + rows) - -#### Documentation -- `analytics/README.md` - Comprehensive analytics documentation -- Use case examples (sales, financial, user analytics) -- Function reference for all modules - -### Changed - -- STATE.scm updated to M18 100% completion -- Added analytics module to working-features - ---- - -## [1.1.0] - 2026-01-12 - -Distributed Mode milestone: **M17 Complete** - -This release adds distributed computing capabilities to Lith. - -### Added - -#### Cluster Coordination (`distributed/src/Lith_Distributed_Cluster.res`) -- Node discovery and membership management -- Node status tracking (Starting, Joining, Active, Leaving, Down) -- Node roles (Leader, Follower, Candidate) -- Cluster state versioning -- JSON serialization for cluster state - -#### Raft Consensus (`distributed/src/Lith_Distributed_Consensus.res`) -- Leader election with term tracking -- Log replication with entry types (Command, Configuration, NoOp) -- Vote request/response handling -- Append entries request/response handling -- Election timeout detection - -#### Data Sharding (`distributed/src/Lith_Distributed_Sharding.res`) -- Consistent hashing with virtual nodes -- Multiple sharding strategies (Hash, Range, Directory) -- Shard status tracking (Initializing, Active, Migrating, Inactive) -- Node-to-shard mapping -- Shard statistics - -#### Data Replication (`distributed/src/Lith_Distributed_Replication.res`) -- Configurable consistency levels (One, Quorum, All, LocalQuorum) -- Replication modes (Synchronous, Asynchronous, SemiSync) -- Replication event queue -- Replica status tracking -- Lag statistics - -#### Documentation -- `distributed/README.md` - Comprehensive distributed mode documentation -- Deployment patterns (single region, multi-region) -- Best practices for cluster sizing and consistency tuning - -### Changed - -- STATE.scm updated to M17 100% completion -- Added distributed module to working-features - ---- - -## [1.0.0] - 2026-01-12 - -**Lith 1.0.0 - First Production Release** - -🎉 The database where the database is part of the story. - -This release marks the first stable, production-ready version of Lith - a narrative-first database where schemas, constraints, migrations, blocks, and journals are treated as narrative artefacts. - -### Highlights - -- **16 Milestones Complete** - All foundational work from M1 (Specification) through M16 (Stabilization) -- **Multi-Protocol API** - REST, gRPC, GraphQL, and WebSocket support -- **Self-Normalizing Engine** - Automatic FD discovery and normalization proposals -- **Proof-Carrying Operations** - Lean 4 integration for verified transformations -- **Production Ready** - Health checks, graceful shutdown, configuration validation - -### Core Features - -#### Storage Layer (Form.Blocks) -- 4 KiB fixed-size blocks with 64-byte headers -- CRC32C integrity verification -- Block types: SUPERBLOCK, DOCUMENT, EDGE, JOURNAL, SCHEMA - -#### Journal System -- Append-only journal with sequence numbering -- Full operation history with inverses -- Crash recovery and replay semantics -- Provenance tracking for audit trails - -#### Query Language (GQL) -- SELECT, INSERT, UPDATE, DELETE operations -- CREATE, DROP for schema management -- EXPLAIN, INTROSPECT for debugging -- WITH PROVENANCE clause for audit context -- Graph traversal syntax (TRAVERSE) - -#### Self-Normalizing Database -- DFD (Depth-First Discovery) for FD detection -- Normal form analysis (1NF through BCNF) -- Three-phase migration: Announce → Shadow → Commit -- Denormalization proposals with rationale - -### Platform Support - -#### API Protocols -| Protocol | Features | -|----------|----------| -| REST | OpenAPI 3.1, full CRUD, health/metrics | -| gRPC | Protobuf, all service methods | -| GraphQL | SDL schema, subscriptions, introspection | -| WebSocket | RFC 6455, graphql-ws, journal streaming | - -#### Client Libraries -| Language | Features | -|----------|----------| -| ReScript | Type-safe, fluent query builder, Deno runtime | -| PHP | PSR-18, PHP 8.1+, Laravel/Symfony integration | - -#### CMS Integrations -| CMS | Type | Sync Modes | -|-----|------|------------| -| Strapi | Plugin | Bidirectional, CMS→Lith, Lith→CMS | -| Directus | Hook Extension | Bidirectional, CMS→Lith, Lith→CMS | -| Ghost | Webhook Server | Bidirectional, CMS→Lith, Lith→CMS | -| Payload | Adapter | Bidirectional, CMS→Lith, Lith→CMS | - -### Quality & Reliability - -#### Testing -- Property-based tests with random generators -- Fuzz testing with 8 mutation strategies -- Integration tests for all CMS plugins -- E2E tests for API and sync scenarios - -#### Performance -- Query plan LRU cache with TTL -- Connection pooling with auto-scaling -- Batch operations with configurable flush -- Prometheus-compatible metrics - -#### Stability -- Type-safe configuration validation -- Component health monitoring -- Graceful shutdown with phased execution -- Production readiness checker - -### Breaking Changes - -None - this is the first stable release. - -### Upgrade Notes - -For projects using pre-1.0 versions: -- API stability is now guaranteed per [VERSIONING.adoc](VERSIONING.adoc) -- Binary format stability guaranteed within 1.x series -- Deprecation warnings will be provided before any breaking changes - -### Thank You - -Lith represents a new approach to databases - one where the database itself becomes part of the story your data tells. Thank you to everyone who contributed to making this release possible. - ---- - -## [0.0.10] - 2026-01-12 - -Final Stabilization milestone: **M16 Complete - Ready for 1.0.0** - -This release completes Milestone M16, the final stabilization before the 1.0.0 production release. - -### Added - -#### Configuration Validation (`stability/src/Lith_Stability_Config.res`) -- Type-safe configuration schema -- Environment-specific validation rules -- Production security enforcement (API key, CORS) -- Environment variable loading -- Validation error formatting - -#### Health Checks (`stability/src/Lith_Stability_Health.res`) -- Component health monitoring (Healthy/Degraded/Unhealthy) -- Extensible health check registry -- Built-in checks: memory, storage, bridge -- Latency tracking per component -- JSON export for Kubernetes probes - -#### Graceful Shutdown (`stability/src/Lith_Stability_Shutdown.res`) -- Coordinated shutdown sequence -- Priority-based handler execution -- Four shutdown phases: DrainConnections → FlushBuffers → CloseResources → Terminated -- Configurable timeout -- Status reporting - -#### Production Readiness (`stability/src/Lith_Stability_Readiness.res`) -- Pre-flight checks across 5 categories: - - Security: API key, CORS, TLS - - Performance: Pool size, query cache - - Reliability: Health endpoint, graceful shutdown - - Observability: Metrics, tracing, logging - - Configuration: Environment setting -- Severity levels (Critical/Warning/Info) -- Human-readable report formatting - -#### Documentation -- `stability/README.md` - Comprehensive stability documentation -- Kubernetes integration examples (liveness/readiness probes) -- Production deployment best practices - -### Changed - -- STATE.scm updated to M16 100% completion -- Added stability module to working-features - ---- - -## [0.0.9] - 2026-01-12 - -Performance Optimization milestone: **M15 Complete** - -This release completes Milestone M15, delivering production-ready performance features. - -### Added - -#### Query Plan Cache (`perf/src/Lith_Perf_Cache.res`) -- LRU cache with TTL-based expiration -- Configurable max size and TTL -- Cache hit/miss statistics -- Thread-safe entry management -- Automatic stale entry cleanup - -#### Connection Pool (`perf/src/Lith_Perf_Pool.res`) -- Connection pooling with min/max sizing -- Idle timeout and automatic cleanup -- Acquire timeout with waiting queue -- Pool statistics (total, idle, in-use, waiting) -- Connection health management - -#### Batch Operations (`perf/src/Lith_Perf_Batch.res`) -- Batch insert, update, delete operations -- Configurable batch size and flush intervals -- Retry on failure with max retries -- Auto-flush when batch size limit reached -- Error tracking per operation - -#### Performance Metrics (`perf/src/Lith_Perf_Metrics.res`) -- Prometheus-compatible metric export -- Counter and Gauge metric types -- Timer context for latency measurement -- Pre-defined metrics: - - `lith_query_total` - Total queries executed - - `lith_query_latency_ms` - Last query latency - - `lith_cache_hits_total` / `lith_cache_misses_total` - - `lith_connection_pool_size` / `lith_active_connections` - - `lith_batch_size` / `lith_errors_total` - -#### Documentation -- `perf/README.md` - Comprehensive performance module documentation -- Best practices for caching, pooling, batching, and monitoring -- Configuration reference for all modules - -### Changed - -- STATE.scm updated to M15 100% completion -- Added performance module to working-features - ---- - -## [0.0.8] - 2026-01-12 - -Testing & Verification milestone: **M14 Complete** - -This release completes Milestone M14, delivering a comprehensive testing framework. - -### Added - -#### Property-Based Tests (`tests/property/`) -- Random GQL statement generators -- Structural property verification -- Configurable iterations and seed -- Property test runner with result tracking - -#### Fuzz Testing (`tests/fuzz/`) -- Multiple mutation strategies (BitFlip, ByteFlip, Dictionary, etc.) -- Corpus-based fuzzing with seed inputs -- Crash and interesting input detection -- GQL parser fuzz targets - -#### Integration Tests (`tests/integration/`) -- Strapi plugin tests (9 test cases) -- Directus extension tests (7 test cases) -- Ghost webhook tests (8 test cases) -- Payload adapter tests (9 test cases) -- Mock HTTP client for isolated testing - -#### E2E Tests (`tests/e2e/`) -- API suite (health, CRUD, queries, introspection) -- Sync suite (create, update, delete, provenance, bidirectional) -- HTTP client utilities for Deno - -#### Test Documentation -- `tests/README.md` - Comprehensive test suite documentation -- Quick start guide for all test categories -- CI integration examples -- Configuration reference - -### Changed - -- STATE.scm updated to M14 100% completion -- Added test components to working-features - ---- - -## [0.0.7] - 2026-01-12 - -CMS Integrations milestone: **M13 Complete** - -This release completes Milestone M13, delivering official CMS integration plugins for popular headless CMS platforms. - -### Added - -#### CMS Integrations (`integrations/`) - -- **Strapi Plugin** (`integrations/strapi/`) - - Strapi v4/v5 plugin written in ReScript - - Real-time content sync to Lith - - Lifecycle hooks: afterCreate, afterUpdate, afterDelete - - Collection mapping with configurable sync modes - - Field exclusion support for sensitive data - - Provenance metadata for audit trails - - Types: StrapiContext, ContentTypeConfig, SyncMode - -- **Directus Extension** (`integrations/directus/`) - - Hook extension for Directus CMS - - Action handlers for items.create, items.update, items.delete - - Environment-based configuration - - Selective collection sync via LITH_SYNC_COLLECTIONS - - Lith client with GQL query execution - -- **Ghost Integration** (`integrations/ghost/`) - - Webhook server for Ghost CMS (Deno runtime) - - Event types: post.published, post.updated, post.deleted - - Page and member events support - - HMAC signature verification - - Configurable collection mappings - - Docker deployment support - -- **Payload CMS Adapter** (`integrations/payload/`) - - Plugin for Payload CMS - - Collection hooks: afterChange, afterDelete - - Field exclusion configuration - - Localized field support (nested locale objects) - - TypeScript type definitions included - -#### Sync Modes (All Integrations) - -| Mode | Description | -|------|-------------| -| `bidirectional` | Sync changes both ways | -| `cms-to-lith` | Only sync CMS changes to Lith | -| `lithoglyph-to-cms` | Only sync Lith changes to CMS | - -#### Provenance Tracking - -All integrations add provenance metadata to Lith: -```json -{ - "actor": "strapi-plugin", - "rationale": "Auto-sync from Strapi create event", - "source": "strapi", - "model": "article", - "action": "create", - "timestamp": "2026-01-12T10:30:00Z" -} -``` - -#### Documentation - -- `integrations/README.md` - Comprehensive integration overview -- Quick start examples for all four CMS platforms -- Environment variable reference -- Architecture diagram - -### Changed - -- STATE.scm updated to M13 100% completion -- Added CMS integration components to working-features - ---- - -## [0.0.6] - 2026-01-12 - -Language Bindings milestone: **M12 Complete** - -This release completes Milestone M12, delivering official client libraries for ReScript and PHP. - -### Added - -#### Client Libraries - -- **ReScript Client** (`clients/rescript/`) - - Type-safe client for Deno runtime - - Full type definitions: Provenance, QueryResult, Collection, JournalEntry - - Fluent query builder with type-safe WHERE clauses - - Filter expressions: Field, And, Or, Not - - Support for all GQL operations (SELECT, INSERT, UPDATE, DELETE) - - Collection management (list, create, delete) - - Journal access with filtering - - Normalization operations (discover dependencies, analyze normal form) - - Migration operations (start, commit) - - Health check endpoint - - Environment-based configuration (LITH_URL, LITH_API_KEY) - - API key and Bearer token authentication - -- **PHP Client** (`clients/php/`) - - PSR-18 HTTP client compatible - - PHP 8.1+ with strict types - - Full type definitions as final readonly classes - - Enums: CollectionType, JournalOperation, NormalForm, ConfidenceLevel, MigrationPhase, HealthStatus - - Fluent query builders: QueryBuilder, InsertBuilder, UpdateBuilder, DeleteBuilder - - Filter classes: FieldFilter, AndFilter, OrFilter, NotFilter with CompareOp enum - - All comparison operators: =, !=, <, <=, >, >=, LIKE, IN - - Framework integration examples: Laravel, Symfony - - LithException with error codes and details - - Environment-based configuration - -#### SDK Generator (`tools/sdk-gen/`) - -- API specification in ReScript (`ApiSpec.res`) - - Full Lith REST API model - - Type definitions, endpoints, parameters - - HTTP methods, request/response types - -- Code generators - - `ReScriptGen.res` - Generate ReScript client code - - `PhpGen.res` - Generate PHP client code - -- CLI entry point (`Main.res`) - - `deno task gen:rescript` - Generate ReScript SDK - - `deno task gen:php` - Generate PHP SDK - -#### Documentation - -- `clients/README.md` - Comprehensive client library documentation - - Quick start examples for both languages - - Query builder usage patterns - - Filter expression examples - - Authentication configuration - - Error handling patterns - -### Changed - -- STATE.scm updated to M12 100% completion -- Tech stack now includes "clients" section with ReScript and PHP - ---- - -## [0.0.5] - 2026-01-12 - -Multi-Protocol API Server milestone: **M11 Complete** - -This release completes Milestone M11, delivering a production-ready multi-protocol API server with full Form.Bridge FFI integration. - -### Added - -#### Multi-Protocol API Server -- **REST API** (`api/src/rest.zig`) - - OpenAPI 3.1 specification compliance - - Full CRUD endpoints for collections, documents, and queries - - Health and metrics endpoints - - Wired to Form.Bridge FFI for real database operations - -- **gRPC API** (`api/src/grpc.zig`) - - Protocol Buffer serialization/deserialization - - Full protobuf encoder with varint, tag/wire type handling - - Protobuf decoder for message parsing - - All service methods wired to Form.Bridge FFI - - Support for Query, ListCollections, GetCollection, CreateCollection - - Support for GetJournal, DiscoverDependencies, AnalyzeNormalForm - - Migration operations (StartMigration, GetMigrationStatus) - -- **GraphQL API** (`api/src/graphql.zig`) - - GraphQL SDL schema with full type system - - Query, Mutation, and Subscription support - - GraphiQL UI for exploration - - Introspection support - - WebSocket integration for subscriptions - -- **WebSocket Support** (`api/src/websocket.zig`) - - RFC 6455 compliant WebSocket implementation - - WebSocket upgrade handling with SHA-1 accept key - - Frame encoding/decoding (text, binary, ping, pong, close) - - graphql-ws protocol for GraphQL subscriptions - - Subscription management with connection state - - Journal streaming subscription type - -- **Form.Bridge Integration** (`api/src/bridge_client.zig`) - - FFI bindings to core Lith engine - - CBOR encoding for all operations - - Graceful degraded mode when bridge unavailable - - Health check, query execution, collection management - -#### Integration Tests (`api/src/integration_tests.zig`) -- REST API endpoint tests -- gRPC protobuf encoder/decoder tests -- gRPC frame encoding tests -- GraphQL request parsing tests -- WebSocket accept key computation tests (RFC 6455 compliance) -- WebSocket frame encoding tests -- WebSocket subscription message tests -- Bridge client integration tests - -#### Ecosystem Coordination -- **UNIFIED-ROADMAP.scm** - Cross-repo roadmap for MVP 1.0.0 - - Dependency graph for Lith, FQLdt, Studio, Debugger - - Critical path phases P1-P3 - - Post-MVP roadmap (1.1.0, 1.2.0, 2.0.0) - - Success metrics and quality gates - -### Changed -- Updated STATE.scm to M11 100% completion -- All API handlers now use Form.Bridge FFI instead of mock responses -- Build configuration includes all new modules in test suite - ---- - -## [0.0.4] - 2026-01-12 - -MVP Completion milestone: **Form.Runtime + Form.Normalizer Complete** - -This release completes Milestones M8-M10, delivering a fully functional query engine and self-normalizing database capabilities. - -### Added - -#### Form.Runtime (M8) - Query Engine -- **GQL Parser** (`core-factor/gql/gql.factor`) - - Full PEG-based parser for GQL statements - - Support for SELECT, INSERT, UPDATE, DELETE, CREATE, DROP - - EXPLAIN, INTROSPECT statements - - WHERE clause with comparison operators - - LIMIT/OFFSET pagination - - Graph traversal (TRAVERSE) syntax - - WITH PROVENANCE clause - -- **Query Planner** - - Cost-based query planning - - Step types: scan, project, limit, traverse, insert, update, delete - - Plan optimization for filtered queries - - Rationale generation for plan decisions - -- **Query Executor** - - In-memory and pluggable persistent storage - - Full CRUD operations - - Filter evaluation engine - - Introspection commands (SCHEMA, CONSTRAINTS, COLLECTIONS, JOURNAL) - -- **EXPLAIN Modes** - - `EXPLAIN` - Show query plan - - `EXPLAIN ANALYZE` - Execute and report timing - - `EXPLAIN VERBOSE` - PostgreSQL-style readable plan output - - `EXPLAIN ANALYZE VERBOSE` - Combined timing and verbose output - -#### Form.Normalizer (M9) - Self-Normalizing Engine -- **FD Discovery** (`normalizer/factor/fd-discovery.factor`) - - DFD (Depth-First Discovery) algorithm implementation - - Configurable sampling and confidence thresholds - - Three-tier confidence classification (high/medium/low) - - Attribute partition refinement - - Discovered FD validation - -- **Normal Form Analysis** - - 1NF through BCNF detection - - Violation identification with explanations - - Prime attribute detection - - Key inference from functional dependencies - -- **Denormalization Proposals** - - Automatic denormalization suggestion generation - - Join-based vs materialized view approaches - - Migration path generation - -- **Three-Phase Migration Framework** (`normalizer/factor/migration.factor`) - - Announce phase: Signal intent, generate rewrite rules - - Shadow phase: Dual-write to old and new schemas - - Commit phase: Complete migration, remove compatibility views - - Query rewriting during migration - - Rollback support (abort migration) - - Migration state tracking and narrative generation - -#### Lean4 Integration -- **Bridge.lean** - FFI bindings with CBOR encoding - - Status codes matching Zig ABI - - FD proof encoding/decoding - - Normalization proof encoding - - Verification API - -- **Proofs.lean** - Proof-carrying transformations - - VerifiedFD, VerifiedNormalizationStep, VerifiedDenormalizationStep - - VerifiedMigration with phase tracking - - Journal packaging for proof blobs - - Round-trip verification support - -- **lakefile.toml** - Lean4 project configuration - - Dependencies on Std, Mathlib4 - - Build configuration for FormNormalizer library - -#### Production Hardening (M10) -- **Seam Tests** (`core-factor/gql/seam-tests.factor`) - - End-to-end pipeline validation: Parser → Planner → Executor → Normalizer - - EXPLAIN correlation tests - - Error propagation tests - - Large dataset stress tests - -- **Benchmarks** (`core-factor/gql/benchmarks.factor`) - - Parser performance benchmarks - - Planner performance benchmarks - - Executor benchmarks (SELECT, INSERT, UPDATE with varying data sizes) - - FD discovery benchmarks - - Normal form analysis benchmarks - - Full pipeline benchmarks - - Memory estimation utilities - - Quick benchmark for CI regression detection - -- **Storage Backend** (`core-factor/gql/storage-backend.factor`) - - Pluggable storage abstraction - - Memory backend (default, for testing) - - Bridge backend (persistent storage via Form.Bridge) - - Runtime backend selection - -#### Ecosystem Alignment -- Updated ECOSYSTEM.scm with alignment status for gql-dt and lithoglyph-debugger -- Cross-repo STATE.scm synchronization with integration points -- Documented FFI compatibility (CBOR proof blobs) -- Identified alignment gaps for future work - -### Changed -- Executor now uses pluggable storage backend instead of direct hash table -- STATE.scm updated to Phase 8: MVP Complete (85% overall completion) -- All milestones M1-M10 now at 100% - -### Migration Tests -- Comprehensive test suite for three-phase migration lifecycle -- Phase transition validation -- Rewrite rule generation tests -- Compatibility view tests -- Error handling and recovery tests - ---- - -## [0.0.3] - 2026-01-12 - -Documentation milestone release: **Complete Documentation Suite** - -This release completes Milestone M7, providing comprehensive documentation for production deployment and integration. - -### Added -- Comprehensive QUICKSTART.adoc tutorial (15-minute guide with full examples) -- Complete VERSIONING.adoc stability policy document -- Complete documentation suite in `docs/`: - - `docs/DEPLOYMENT.adoc` - Production deployment guide (Docker, Kubernetes, systemd) - - `docs/SECURITY-AUTH.adoc` - Authentication, authorization, and security hardening - - `docs/API-REFERENCE.adoc` - Complete programmatic interface reference (Form.Bridge FFI, GQL API) - - `docs/MIGRATION-FROM-RDBMS.adoc` - PostgreSQL/MySQL/SQLite migration guide with type mappings - - `docs/OBSERVABILITY.adoc` - Logging, Prometheus metrics, OpenTelemetry tracing, Grafana dashboards - - `docs/INTEGRATION-PATTERNS.adoc` - Message queues, search engines, analytics, AI/ML pipelines - -### Changed -- License updated from MPL-2.0 to Palimpsest-MPL 1.0 (PMPL-1.0) -- README.adoc reorganized with comprehensive Documentation section - -### Fixed -- License badge now correctly shows PMPL-1.0 -- Fixed typo in Palimpsest link (licence → license) - ---- - -## [0.0.2] - 2026-01-11 - -Major milestone release: **Core Specifications Complete + PoC Implementation** - -This release completes Milestones M1-M6, establishing Lith as a functional proof-of-concept. - -### Added - -#### Core Specifications (M1) -- **GQL Language Specification** (`spec/gql.adoc`) - - Complete EBNF grammar for PoC subset - - 10 example queries covering all operations - - Document, edge, schema, and introspection operations - - Provenance syntax (`WITH PROVENANCE {...}`) - -- **GQL Dependent Types Specification** (`spec/gql-dependent-types.md`) - - Full FQLdt specification with Lean 4 integration - - Compile-time query verification - - Proof-carrying schema evolution - - Type-level encoding of database constraints - -- **Self-Normalizing Database Specification** (`spec/self-normalizing.adoc`) - - Automatic functional dependency discovery (DFD/TANE/FDHits) - - Normal form predicates (1NF through BCNF) - - Proof-carrying normalization decisions - - Narrative explanations for all schema changes - -- **Block Format Specification** (`spec/blocks.adoc`) - - 4 KiB fixed-size blocks with 64-byte headers - - Block types: SUPERBLOCK, DOCUMENT, EDGE, JOURNAL, SCHEMA, etc. - - CRC32C checksums for integrity - - Compression and encryption flags - -- **Journal Format Specification** (`spec/journal.adoc`) - - Append-only journal with sequence numbers - - Full operation history with inverses - - Crash recovery and replay semantics - -- **Cloud Storage Specification** (`spec/cloud-storage.adoc`) - - Object storage integration patterns - - Tiered storage for hot/warm/cold data - -- **GQL Design Philosophy** (`spec/gql-philosophy.adoc`) - - Narrative-first query design - - Comparison with SQL philosophy - - Constraints as ethics - -#### Forth Implementation (M2-M5) -- **Form.Blocks** (`core-forth/src/lithoglyph-blocks.fs`) - - Fixed-size block storage layer - - Block header structure with magic, version, type, checksums - - Memory buffer management - - CRC32C implementation (Castagnoli polynomial) - -- **Form.Journal** (`core-forth/src/lithoglyph-journal.fs`) - - Append-only journal implementation - - Sequence numbering - - Operation logging with inverses - - Crash recovery primitives - -- **Form.Model** (`core-forth/src/lithoglyph-model.fs`) - - Document collection support - - Edge collection support - - Schema metadata storage - - Constraint storage - -- **Test Suite** (`core-forth/test/`) - - Block operations tests - - Journal operations tests - - Model layer tests - -#### Documentation -- Architecture guide (`ARCHITECTURE.adoc`) -- Roadmap (`ROADMAP.adoc`) -- Philosophy document (`PHILOSOPHY.adoc`) -- Contributing guidelines (`CONTRIBUTING.adoc`) -- Maintainers list (`MAINTAINERS.adoc`) - -#### Ecosystem Integration -- Related projects documentation: - - Lith Studio (zero-friction GUI) - - Lith Debugger (proof-carrying debugger) - - FormBase (Airtable alternative) - - Zotero-Lith (reference manager) - - FQLdt (dependently-typed GQL) - -#### Machine-Readable Artefacts -- 6SCM files for AI agent integration: - - `STATE.scm` - Project state tracking - - `META.scm` - Architecture decisions - - `ECOSYSTEM.scm` - Ecosystem position - - `PLAYBOOK.scm` - Operational runbook - - `AGENTIC.scm` - AI interaction patterns - - `NEUROSYM.scm` - Neurosymbolic config - -### Changed -- Eliminated C dependency in favor of Zig-only ABI (Form.Bridge) -- Consolidated FQLdt specification into single comprehensive document -- Clarified that GQL = Lith Query Language (not "forms" query language) - -### Security -- Fixed workflow security issues (ERR-WF-008, ERR-WF-009) -- Updated actions/cache SHA from v2 to v4 - ---- - -## [0.0.1] - 2026-01-03 - -Initial release: **Repository Initialization** - -### Added - -#### Repository Structure -- Initial repository setup following RSR (Rhodium Standard Repositories) pattern -- Standard hyperpolymath/mustfile structure -- RSR enforcement workflows - -#### Documentation Framework -- README.adoc with project overview -- Core thesis: "Schemas, constraints, migrations, blocks, and journals are narrative artefacts" -- Primary values table (Auditability > Performance, Meaning > Features, etc.) -- Target domains: investigative journalism, governance, agentic ecosystems, archives -- Layer architecture diagram - -#### Licensing -- MPL-2.0 base license -- Palimpsest philosophy notice (ethical open source) - -#### CI/CD -- GitHub Actions workflows for quality enforcement -- Casket-SSG GitHub Pages workflow -- Security scanning workflows - ---- - -## Version History Summary - -| Version | Date | Milestone | Key Features | -|---------|------|-----------|--------------| -| 2.0.0 | 2026-01-12 | **Feature-Complete** | All 18 milestones, distributed + analytics | -| 1.2.0 | 2026-01-12 | M18 Complete | Aggregations, time series, window functions, exports | -| 1.1.0 | 2026-01-12 | M17 Complete | Cluster, consensus, sharding, replication | -| 1.0.0 | 2026-01-12 | **Production** | First stable release, all M1-M16 complete | -| 0.0.10 | 2026-01-12 | M16 Complete | Config validation, health checks, graceful shutdown, readiness | -| 0.0.9 | 2026-01-12 | M15 Complete | Query cache, connection pool, batch ops, metrics | -| 0.0.8 | 2026-01-12 | M14 Complete | Property tests, fuzz testing, integration tests, E2E tests | -| 0.0.7 | 2026-01-12 | M13 Complete | Strapi, Directus, Ghost, Payload CMS integrations | -| 0.0.6 | 2026-01-12 | M12 Complete | ReScript client, PHP client, SDK generator | -| 0.0.5 | 2026-01-12 | M11 Complete | Multi-protocol API server, WebSocket subscriptions | -| 0.0.4 | 2026-01-12 | M8-M10 Complete | Query engine, normalizer, production hardening | -| 0.0.3 | 2026-01-12 | M7 Complete | Complete documentation suite, PMPL-1.0 license | -| 0.0.2 | 2026-01-11 | M1-M6 Complete | Full specs, Forth PoC, documentation | -| 0.0.1 | 2026-01-03 | Repository Init | Structure, licensing, CI/CD | - -## Upgrade Notes - -### Upgrading to 0.0.3 - -No breaking changes from 0.0.2. This release adds comprehensive documentation and changes the license to PMPL-1.0. - -### Upgrading to 0.0.2 - -No breaking changes from 0.0.1. This release adds specifications and implementation. - -### Pre-1.0 Warning - -Lith is in pre-1.0 development. APIs, formats, and interfaces may change without deprecation warnings. See [VERSIONING.adoc](VERSIONING.adoc) for stability guarantees. - ---- - -## Links - -[Unreleased]: https://github.com/hyperpolymath/lithoglyph/compare/v2.0.0...HEAD -[2.0.0]: https://github.com/hyperpolymath/lithoglyph/compare/v1.2.0...v2.0.0 -[1.2.0]: https://github.com/hyperpolymath/lithoglyph/compare/v1.1.0...v1.2.0 -[1.1.0]: https://github.com/hyperpolymath/lithoglyph/compare/v1.0.0...v1.1.0 -[1.0.0]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.10...v1.0.0 -[0.0.10]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.9...v0.0.10 -[0.0.9]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.8...v0.0.9 -[0.0.8]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.7...v0.0.8 -[0.0.7]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.6...v0.0.7 -[0.0.6]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.5...v0.0.6 -[0.0.5]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.4...v0.0.5 -[0.0.4]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.3...v0.0.4 -[0.0.3]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.2...v0.0.3 -[0.0.2]: https://github.com/hyperpolymath/lithoglyph/compare/v0.0.1...v0.0.2 -[0.0.1]: https://github.com/hyperpolymath/lithoglyph/releases/tag/v0.0.1 - -## Related Documentation - -- [VERSIONING.adoc](VERSIONING.adoc) - Stability policy and version guarantees -- [ROADMAP.adoc](ROADMAP.adoc) - Planned features and milestones -- [QUICKSTART.adoc](QUICKSTART.adoc) - Getting started guide -- [ARCHITECTURE.adoc](ARCHITECTURE.adoc) - Technical architecture - -## Changelog Conventions - -This changelog follows these conventions: - -- **Added** - New features -- **Changed** - Changes to existing functionality -- **Deprecated** - Features that will be removed in future versions -- **Removed** - Features that have been removed -- **Fixed** - Bug fixes -- **Security** - Security-related changes - -Each release includes: -- Summary of the milestone achieved -- Detailed list of changes by category -- Breaking changes highlighted (when applicable) -- Upgrade notes (when applicable) diff --git a/lithoglyph/CODE_OF_CONDUCT.md b/lithoglyph/CODE_OF_CONDUCT.md deleted file mode 100644 index ab314f8c..00000000 --- a/lithoglyph/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,327 +0,0 @@ -# Code of Conduct - - - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in Nextgen Databases a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, colour, religion, or sexual identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. - -We recognise that a thriving open source community requires **psychological safety** — an environment where people can contribute, ask questions, make mistakes, and learn without fear of ridicule or retaliation. - ---- - -## Our Standards - -### Expected Behaviour - -The following behaviours contribute to a positive environment: - -**Communication** -- Using welcoming and inclusive language -- Being respectful of differing viewpoints and experiences -- Giving and gracefully accepting constructive feedback -- Assuming good intent while addressing impact -- Communicating clearly and patiently, especially with newcomers - -**Collaboration** -- Focusing on what is best for the community -- Showing empathy and kindness toward other community members -- Being collaborative rather than competitive -- Mentoring and supporting less experienced contributors -- Celebrating others' contributions and successes - -**Professionalism** -- Accepting responsibility and apologising to those affected by our mistakes -- Learning from the experience and avoiding repetition -- Respecting others' time and attention -- Staying on topic in project spaces -- Following project guidelines and conventions - -**Accessibility** -- Using plain language and avoiding unnecessary jargon -- Providing alt text for images and transcripts for audio/video -- Being patient with those using assistive technologies -- Accommodating different communication styles and needs -- Recognising that not everyone communicates the same way - -### Unacceptable Behaviour - -The following behaviours are considered harassment and are unacceptable: - -**Harassment** -- The use of sexualised language or imagery, and sexual attention or advances of any kind -- Trolling, insulting or derogatory comments, and personal or political attacks -- Public or private harassment -- Deliberate intimidation, stalking, or following (online or in-person) -- Unwelcome physical contact or simulated physical contact (e.g., emoji) -- Sustained disruption of talks, events, or online discussions - -**Discrimination** -- Discriminatory jokes and language -- Posting or threatening to post others' personally identifying information ("doxing") -- Advocating for, or encouraging, any of the above behaviour -- Microaggressions — subtle, often unintentional, discriminatory comments or actions - -**Professional Misconduct** -- Publishing others' private information without explicit permission -- Misrepresenting affiliation or contributions -- Plagiarism or claiming credit for others' work -- Retaliating against anyone who reports a Code of Conduct violation -- Other conduct which could reasonably be considered inappropriate in a professional setting - -### Grey Areas - -Some situations require judgement. When uncertain: - -- **Intent vs Impact**: Good intentions do not excuse harmful impact. Focus on making things right. -- **Power Dynamics**: Those with more power (maintainers, employers, experienced contributors) must be especially mindful of their impact. -- **Cultural Differences**: What's acceptable varies by culture. When in doubt, err on the side of caution and ask. -- **Humour**: Jokes at others' expense are rarely funny to everyone. Punch up, not down. - ---- - -## Scope - -This Code of Conduct applies within all community spaces, including: - -**Online Spaces** -- Repository discussions, issues, and pull/merge requests -- Project chat channels (Matrix, Discord, Slack, IRC) -- Mailing lists and forums -- Social media when representing the project -- Video calls and virtual meetings - -**In-Person Spaces** -- Conferences, meetups, and events -- Workshops and training sessions -- Any gathering where you represent the project - -**Representation** -This Code of Conduct also applies when an individual is officially representing the community in public spaces. Examples include: - -- Using an official project email address -- Posting via an official social media account -- Acting as an appointed representative at an event -- Speaking on behalf of the project - ---- - -## Enforcement - -### Reporting - -If you experience or witness unacceptable behaviour, or have any other concerns, please report it as soon as possible. - -**How to Report** - -| Method | Details | Best For | -|--------|---------|----------| -| **Email** | {{CONDUCT_EMAIL}} | Detailed reports, sensitive matters | -| **Private Message** | Contact any maintainer directly | Quick questions, minor issues | -| **Anonymous Form** | [Link to form if available] | When you need anonymity | - -**What to Include** - -- Your contact information (unless anonymous) -- Names/usernames of those involved -- Description of what happened -- When and where it occurred -- Any witnesses -- Any supporting evidence (screenshots, links) -- How you would like us to respond (if you have a preference) - -**What Happens Next** - -1. You will receive acknowledgment within **{{RESPONSE_TIME}}** -2. The {{CONDUCT_TEAM}} will review the report -3. We may ask for additional information -4. We will determine appropriate action -5. We will inform you of the outcome (respecting others' privacy) - -### Confidentiality - -All reports will be handled with discretion: - -- Reporter identity is protected by default -- Details are shared only with those who need to know -- We will ask before naming you in any communication -- Anonymous reports are accepted and investigated - -### Conflicts of Interest - -If a {{CONDUCT_TEAM}} member is involved in an incident: - -- They will recuse themselves from the process -- Another maintainer or external party will handle the report -- We will disclose any potential conflicts - ---- - -## Enforcement Guidelines - -The {{CONDUCT_TEAM}} will follow these guidelines in determining consequences: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behaviour deemed unprofessional or unwelcome. - -**Consequence**: A private, written warning providing clarity around the nature of the violation and an explanation of why the behaviour was inappropriate. A public apology may be requested. - -**Duration**: Immediate - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of actions. - -**Consequence**: A warning with consequences for continued behaviour. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. - -**Duration**: 1-4 weeks - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including sustained inappropriate behaviour. - -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. - -**Duration**: 1-6 months - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behaviour, harassment of an individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the community. - -**Duration**: Permanent (with appeal rights after 12 months) - -### Enforcement Across Perimeters - -For contributors with elevated access (Perimeter 2 or 1): - -| Level | Additional Consequence | -|-------|----------------------| -| Correction | Noted in contributor record | -| Warning | Access privileges may be temporarily reduced | -| Temporary Ban | Access reduced to Perimeter 3 for ban duration | -| Permanent Ban | All access revoked | - ---- - -## Appeals - -If you believe an enforcement decision was made in error: - -1. **Wait 7 days** after the decision (cooling-off period) -2. **Email** {{CONDUCT_EMAIL}} with subject line "Appeal: [Original Report ID]" -3. **Explain** why you believe the decision should be reconsidered -4. **Provide** any new information not previously available - -**Appeals Process** - -- Appeals are reviewed by a different {{CONDUCT_TEAM}} member than the original -- You will receive a response within 14 days -- The appeals decision is final -- You may only appeal once per incident - -**Grounds for Appeal** - -- Procedural errors in the original investigation -- New evidence not previously available -- Disproportionate response to the violation -- Misunderstanding of facts - ---- - -## Supporting Those Who Report - -We are committed to supporting those who report violations: - -**We Will** -- Believe and take all reports seriously -- Respect your privacy and confidentiality preferences -- Keep you informed of progress (if you wish) -- Take steps to protect you from retaliation -- Provide resources if you need support - -**We Will Not** -- Require you to confront the person directly -- Dismiss reports without investigation -- Reveal your identity without consent -- Tolerate retaliation against reporters -- Rush you to make decisions - ---- - -## Prevention - -Beyond enforcement, we actively work to prevent issues: - -**Onboarding** -- All contributors are expected to read this Code of Conduct -- Perimeter 2 applicants must confirm they've read and understood it -- Maintainers receive additional training on enforcement - -**Culture** -- We model the behaviour we expect -- We intervene early when we see potential issues -- We thank people for positive contributions -- We create opportunities for diverse voices - -**Review** -- This Code of Conduct is reviewed annually -- Community feedback is welcomed -- Changes are communicated clearly - ---- - -## Acknowledgments - -This Code of Conduct is adapted from: - -- [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1 -- [Django Code of Conduct](https://www.djangoproject.com/conduct/) -- [Rust Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct) -- [Python Community Code of Conduct](https://www.python.org/psf/conduct/) - -We thank these communities for their leadership in creating welcoming spaces. - ---- - -## Questions? - -If you have questions about this Code of Conduct: - -- Open a [Discussion](https://github.com/hyperpolymath/nextgen-databases/discussions) (for general questions) -- Email {{CONDUCT_EMAIL}} (for private questions) -- Contact any maintainer directly - ---- - -## Summary - -**Be kind. Be respectful. Be collaborative.** - -We're all here because we care about this project. Let's make it a place where everyone can do their best work. - ---- - -Last updated: 2026 · Based on Contributor Covenant 2.1 diff --git a/lithoglyph/CONTRIBUTING.adoc b/lithoglyph/CONTRIBUTING.adoc deleted file mode 100644 index 9ef7ef1b..00000000 --- a/lithoglyph/CONTRIBUTING.adoc +++ /dev/null @@ -1,76 +0,0 @@ -= Contributing to Lith -:toc: - -Thank you for your interest in contributing to Lith! - -== Code of Conduct - -Be respectful, inclusive, and constructive. We value diverse perspectives and collaborative problem-solving. - -== Getting Started - -1. Read link:README.adoc[README.adoc] for project overview -2. Read link:lithoglyph.scm[lithoglyph.scm] for detailed specification -3. Review link:META.scm[META.scm] for architecture decisions -4. Check link:STATE.scm[STATE.scm] for current project state - -== How to Contribute - -=== Reporting Issues - -* Search existing issues first -* Include reproduction steps -* Specify version and environment - -=== Proposing Changes - -1. Open an issue to discuss the change -2. Reference relevant open questions from `lithoglyph.scm` -3. Fork and create a feature branch -4. Follow the coding conventions below -5. Submit a pull request - -=== Closing Open Questions - -If your contribution addresses an open question from `lithoglyph.scm`: - -1. Add a decision record to the `(decisions ...)` section -2. Document rationale and impacts -3. Update affected specifications - -== Coding Conventions - -=== Forth (core-forth/) - -* One word per line for complex definitions -* Document stack effects: `( before -- after )` -* Keep words small and composable - -=== Zig (core-zig/) - -* Follow Zig style guide -* Explicit error handling -* No hidden allocations - -=== Factor (core-factor/) - -* Use vocabularies to organize code -* Document stack effects -* Prefer quotations over complex control flow - -=== Elixir (control-plane/) - -* Standard Elixir formatting (`mix format`) -* Dialyzer type specs -* OTP patterns for supervision - -== Pull Request Process - -1. Ensure all tests pass -2. Update documentation as needed -3. Add entries to CHANGELOG if applicable -4. Request review from maintainers - -== License - -By contributing, you agree that your contributions will be licensed under PMPL-1.0-or-later. diff --git a/lithoglyph/CONTRIBUTING.md b/lithoglyph/CONTRIBUTING.md deleted file mode 100644 index bd6f5a5e..00000000 --- a/lithoglyph/CONTRIBUTING.md +++ /dev/null @@ -1,116 +0,0 @@ -# Clone the repository -git clone https://github.com/hyperpolymath/nextgen-databases.git -cd nextgen-databases - -# Using Nix (recommended for reproducibility) -nix develop - -# Or using toolbox/distrobox -toolbox create nextgen-databases-dev -toolbox enter nextgen-databases-dev -# Install dependencies manually - -# Verify setup -just check # or: cargo check / mix compile / etc. -just test # Run test suite -``` - -### Repository Structure -``` -nextgen-databases/ -├── src/ # Source code (Perimeter 1-2) -├── lib/ # Library code (Perimeter 1-2) -├── extensions/ # Extensions (Perimeter 2) -├── plugins/ # Plugins (Perimeter 2) -├── tools/ # Tooling (Perimeter 2) -├── docs/ # Documentation (Perimeter 3) -│ ├── architecture/ # ADRs, specs (Perimeter 2) -│ └── proposals/ # RFCs (Perimeter 3) -├── examples/ # Examples (Perimeter 3) -├── spec/ # Spec tests (Perimeter 3) -├── tests/ # Test suite (Perimeter 2-3) -├── .well-known/ # Protocol files (Perimeter 1-3) -├── .github/ # GitHub config (Perimeter 1) -│ ├── ISSUE_TEMPLATE/ -│ └── workflows/ -├── CHANGELOG.md -├── CODE_OF_CONDUCT.md -├── CONTRIBUTING.md # This file -├── GOVERNANCE.md -├── LICENSE -├── MAINTAINERS.md -├── README.adoc -├── SECURITY.md -├── flake.nix # Nix flake (Perimeter 1) -└── Justfile # Task runner (Perimeter 1) -``` - ---- - -## How to Contribute - -### Reporting Bugs - -**Before reporting**: -1. Search existing issues -2. Check if it's already fixed in `main` -3. Determine which perimeter the bug affects - -**When reporting**: - -Use the [bug report template](.github/ISSUE_TEMPLATE/bug_report.md) and include: - -- Clear, descriptive title -- Environment details (OS, versions, toolchain) -- Steps to reproduce -- Expected vs actual behaviour -- Logs, screenshots, or minimal reproduction - -### Suggesting Features - -**Before suggesting**: -1. Check the [roadmap](ROADMAP.md) if available -2. Search existing issues and discussions -3. Consider which perimeter the feature belongs to - -**When suggesting**: - -Use the [feature request template](.github/ISSUE_TEMPLATE/feature_request.md) and include: - -- Problem statement (what pain point does this solve?) -- Proposed solution -- Alternatives considered -- Which perimeter this affects - -### Your First Contribution - -Look for issues labelled: - -- [`good first issue`](https://github.com/hyperpolymath/nextgen-databases/labels/good%20first%20issue) — Simple Perimeter 3 tasks -- [`help wanted`](https://github.com/hyperpolymath/nextgen-databases/labels/help%20wanted) — Community help needed -- [`documentation`](https://github.com/hyperpolymath/nextgen-databases/labels/documentation) — Docs improvements -- [`perimeter-3`](https://github.com/hyperpolymath/nextgen-databases/labels/perimeter-3) — Community sandbox scope - ---- - -## Development Workflow - -### Branch Naming -``` -docs/short-description # Documentation (P3) -test/what-added # Test additions (P3) -feat/short-description # New features (P2) -fix/issue-number-description # Bug fixes (P2) -refactor/what-changed # Code improvements (P2) -security/what-fixed # Security fixes (P1-2) -``` - -### Commit Messages - -We follow [Conventional Commits](https://www.conventionalcommits.org/): -``` -(): - -[optional body] - -[optional footer] diff --git a/lithoglyph/Containerfile b/lithoglyph/Containerfile deleted file mode 100644 index ccec7de6..00000000 --- a/lithoglyph/Containerfile +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Lithoglyph Lith — Container build -# Base: Chainguard wolfi (NEVER debian/ubuntu) -# -# Build with: podman build -t lithoglyph-lith -f Containerfile . -# Run with: podman run -p 8080:8080 lithoglyph-lith -# -# Multi-stage build: wolfi-base for compilation, static for minimal runtime. - -# ============================================================================= -# Stage 1: Build — install Zig, compile core-zig bridge and demo-server -# ============================================================================= -FROM cgr.dev/chainguard/wolfi-base:latest AS builder - -# Install build dependencies -RUN apk add --no-cache \ - curl \ - xz \ - tar \ - glibc-dev - -# Install Zig 0.15.2 -RUN curl -L https://ziglang.org/download/0.15.2/zig-linux-x86_64-0.15.2.tar.xz | tar -xJ -C /usr/local && \ - ln -s /usr/local/zig-linux-x86_64-0.15.2/zig /usr/local/bin/zig - -WORKDIR /build - -# Copy source files -COPY core-zig/ /build/core-zig/ -COPY demo-server.zig /build/ - -# Build core-zig using the build system (produces static + shared libs) -WORKDIR /build/core-zig -RUN zig build -Doptimize=ReleaseFast - -# Also build standalone shared library for FFI consumers -RUN zig build-lib src/bridge.zig -dynamic -lc -O ReleaseFast - -# Build demo-server (the API binary) -WORKDIR /build -RUN zig build-exe demo-server.zig -lc -O ReleaseFast - -# ============================================================================= -# Stage 2: Runtime — minimal static image with only the binary -# ============================================================================= -FROM cgr.dev/chainguard/static:latest - -WORKDIR /app - -# Copy only the compiled binary and shared library from builder -COPY --from=builder /build/demo-server /app/ -COPY --from=builder /build/core-zig/libbridge.so /app/ - -# Set library path so the binary can find libbridge.so -ENV LD_LIBRARY_PATH=/app - -# Auth token must be provided at runtime via environment variable. -# Example: podman run -e LITHOGLYPH_AUTH_TOKEN= ... -# See api/src/auth.zig for details. -# ENV LITHOGLYPH_AUTH_TOKEN= (intentionally not set — must be provided) - -# Expose API port -EXPOSE 8080 - -# Run server as the default entrypoint -ENTRYPOINT ["/app/demo-server"] diff --git a/lithoglyph/HONEST-STATUS-2026-02-13.md b/lithoglyph/HONEST-STATUS-2026-02-13.md deleted file mode 100644 index 444fe645..00000000 --- a/lithoglyph/HONEST-STATUS-2026-02-13.md +++ /dev/null @@ -1,143 +0,0 @@ -# Lithoglyph Honest Status — 2026-02-13 (Final) - -## Session Summary - -Two major work phases completed today: - -### Phase A: 7-Phase ABI Plan (Interface Contracts) - -| Phase | Description | Status | -|-------|-------------|--------| -| 1 | Delete template ABI duplicates (`src/abi/`) | Done | -| 2 | Eliminate `believe_me` from `src/Lith/` (10 total) | Done — 0 remain in Lith | -| 3 | Align Idris2 ABI with core-zig bridge (18 FFI declarations) | Done | -| 4 | Generate C header (`generated/abi/bridge.h`) | Done | -| 5 | Update Factor FFI (`storage-backend.factor`) | Done | -| 6 | Expand C FFI integration tests (5→17) | Done | -| 7 | Create ReScript test runners (property + fuzz) | Done | - -### Phase B: Compile Verification and Fixes - -| Component | Before | After | -|-----------|--------|-------| -| **Idris2 ABI** (`src/Lith/`) | 11 typed holes, 10 believe_me | 0 holes, 0 believe_me, compiles clean | -| **core-zig** | Working but no `pub` exports | BUILD + TEST PASS | -| **ffi/zig** | 23 stubs (dead code) | Delegates to core-zig, BUILD + TEST PASS | -| **core-forth** | Working | 17/17 TESTS PASS | -| **beam/native (Zig NIF)** | Stubs ("M10 PoC") | Real FFI calls, BUILD PASS | -| **beam/native_rust** | Lifetime errors, 6 warnings | BUILD PASS, 0 warnings | -| **API layer** | Placeholder responses + old Zig API | Bridge calls wired, main.zig updated (see PENDING) | -| **Subproject ABIs** | 4 believe_me (banned) | Replaced with %foreign prim__callbackToAnyPtr | -| **Zig unsafe casts** | 22 without SAFETY comments | All annotated with `// SAFETY:` | -| **SQL injection** | String concatenation in test gen | Parameterized query builders | -| **Production infra** | None | Containerfile, selur-compose.yml, CI workflow, env-var auth | -| **Test vectors** | Empty dirs | 5 encoding test vectors + 3 ExUnit test files | - ---- - -## What Actually Compiles and Passes Tests - -| Component | Build | Tests | Notes | -|-----------|-------|-------|-------| -| core-zig (bridge + blocks) | PASS | PASS | 19 real functions, WAL commit | -| ffi/zig (delegation layer) | PASS | PASS | Delegates to core-zig | -| core-forth (Forth kernel) | PASS | 17/17 PASS | Block storage, journal, model | -| core-lean (Lean 4 proofs) | PASS | 52 PASS | Normalization proofs | -| Idris2 ABI (src/Lith/) | PASS | N/A | All 3 files type-check clean | -| beam/native (Zig NIF) | PASS | N/A | Real FFI calls to core-zig | -| beam/native_rust (Rust NIF) | PASS (0 warnings) | N/A | Rustler 0.35 NIF | -| core-factor (GQL runtime) | PASS | N/A | GQL parser, planner, executor | - ---- - -## What's Still Broken or Incomplete - -### API Layer (api/) — Pre-existing Zig 0.15.2 Incompatibility - -`api/src/main.zig` has been updated to 0.15.2 `std.net.Server` pattern. -`api/src/rest.zig` still uses old `std.http.Server` API — **83 call sites** need migration. -`api/src/grpc.zig` has the same issue. - -This is a pre-existing problem, not introduced by today's work. The API layer -has real bridge calls wired in (not placeholders), but won't compile until -the HTTP API is migrated. - -### Studio (Tauri) — 11 TODO Commands - -Every backend command in `studio/src-tauri/src/main.rs` returns mock data. -This is a satellite component, not core infrastructure. - -### Naming — Lith/Lith → Lith/Litho (IP Issue) - -**CRITICAL — must be done before any public release.** - -"Form" and "Lith" had an IP claim. Everything must be renamed: -- `Lith` → `LithBD` or `Litho` -- `Lith` → `LithDB` or `Lithoglyph` -- `Form.Bridge` → `Lith.Bridge` -- `LithLayout` → `LithLayout` -- `LithForeign` → `LithForeign` -- `LithBridge` → `LithBridge` -- `GQL` → `LDQL` or `LithQL` -- `lith_*` C symbols → `lg_*` or `lith_*` -- Module directory `src/Lith/` → `src/LithBD/` or `src/Litho/` -- Factor vocabulary `storage-backend` references -- `generated/abi/bridge.h` type prefixes -- All comments, docs, test names - -**Scope:** Entire repo — .idr, .zig, .rs, .res, .factor, .fs, .h, .json, .md, .yml. -Keep "form" only where it literally means HTML/data forms (not the database). - ---- - -## Banned Patterns Status - -| Pattern | Before | After | -|---------|--------|-------| -| `believe_me` in src/Lith/ | 10 | **0** | -| `believe_me` in subproject ABIs | 4 | **0** (replaced with %foreign) | -| Unsafe Zig casts without SAFETY | 22 | **0** (all annotated) | -| SQL injection in test generators | 1 | **0** (parameterized) | -| Hardcoded auth tokens | 1 | **0** (env-var based) | - ---- - -## Production Infrastructure Added - -- `Containerfile` — Multi-stage build, chainguard/static runtime -- `selur-compose.yml` — Podman-compatible orchestration -- `.github/workflows/ci.yml` — SHA-pinned CI (Zig, Forth, Lean, Idris2, Rust) - ---- - -## Proven Library Integration - -The real Proven library exists at `/var$REPOS_DIR/proven/` with 104+ modules -(SafePath, SafeJson, SafeSQL, SafeSchema, SafeBuffer, SafePolicy, etc.) and 89 -language bindings. Lithoglyph should import from this external repo, NOT bundle a copy. - -Current inline implementations in LithBridge.idr (validateDbPath, validateFqlQuery, -parseJsonDocument) are functional but should eventually delegate to Proven. - ---- - -## panic-attack Scan: 22 Weak Points (Pre-Fix Baseline) - -| Severity | Count | -|----------|-------| -| Critical | 5 | -| High | 5 | -| Medium | 8 | -| Low | 4 | - -Many of these have been addressed (believe_me elimination, SQL injection fix, -auth hardening, SAFETY annotations). A re-scan is recommended. - ---- - -## Bottom Line - -The core storage engine, Zig bridge, Forth kernel, Lean proofs, and Idris2 ABI -all compile and pass tests. The BEAM NIFs (both Zig and Rust) compile clean. -The API layer needs Zig 0.15.2 HTTP migration. The Lith→Lith rename is pending. -This is verified scaffolding with a working foundation — not just a blueprint. diff --git a/lithoglyph/Justfile b/lithoglyph/Justfile deleted file mode 100644 index 8f9f580a..00000000 --- a/lithoglyph/Justfile +++ /dev/null @@ -1,169 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# justfile - Just recipes for Lithoglyph -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) - -# Default recipe -default: - @just --list - -# ============================================================ -# BUILD -# ============================================================ - -# Build Zig libraries (static + shared) -build-zig: - cd core-zig && zig build - -# Build shared library for FFI consumers -build-ffi: - cd core-zig && zig build-lib src/bridge.zig -dynamic -lc -O ReleaseFast - -# Build C FFI integration test binary -build-ffi-tests: - cd core-zig && gcc -o test-ffi-integration test-ffi-integration.c -L zig-out/lib -llith_bridge - cd core-zig && gcc -o test-version-only test-version-only.c -L zig-out/lib -llith_bridge - cd core-zig && gcc -o test-db-open test-db-open.c -L zig-out/lib -llith_bridge - -# Build everything -build: build-zig - -# ============================================================ -# TESTS -# ============================================================ - -# Run Zig unit tests (bridge + blocks) -test-zig: - @echo "=== Zig Unit Tests ===" - cd core-zig && zig build test - @echo "Zig tests passed" - -# Run Zig tests directly (faster, no build system) -test-zig-fast: - @echo "=== Zig Direct Tests ===" - cd core-zig && zig test src/bridge.zig - cd core-zig && zig test src/blocks.zig - -# Run Forth block layer tests -test-forth: - @echo "=== Forth Block Tests ===" - cd core-forth/test && gforth test-blocks.fs -e bye - -# Run C FFI integration tests (requires build-zig first) -test-ffi: build-zig - @echo "=== C FFI Integration Tests ===" - cd core-zig && LD_LIBRARY_PATH=zig-out/lib ./test-ffi-integration - -# Run C FFI quick smoke tests -test-ffi-smoke: build-zig - @echo "=== C FFI Smoke Tests ===" - cd core-zig && LD_LIBRARY_PATH=zig-out/lib ./test-version-only - cd core-zig && LD_LIBRARY_PATH=zig-out/lib ./test-db-open - -# Run Factor seam tests -test-factor: - @echo "=== Factor Seam Tests ===" - cd core-factor/gql && factor seam-tests.factor - -# Run ReScript property tests -test-property: - @echo "=== ReScript Property Tests ===" - cd tests/property && deno task test - -# Run ReScript fuzz tests (quick: 1K iterations) -test-fuzz-quick: - @echo "=== ReScript Fuzz Tests (Quick) ===" - cd tests/fuzz && deno task fuzz:quick - -# Run ReScript fuzz tests (standard: 10K iterations) -test-fuzz: - @echo "=== ReScript Fuzz Tests ===" - cd tests/fuzz && deno task fuzz - -# Run ReScript integration tests (requires Deno + ReScript) -test-integration: - @echo "=== ReScript Integration Tests ===" - cd tests/integration && deno task test - -# Run E2E tests (requires running Lith server on :8080) -test-e2e: - @echo "=== E2E Tests (requires server on :8080) ===" - cd tests/e2e && deno task test - -# Run all core tests (no server required) -test: test-zig test-forth test-ffi - -# Run all tests including ReScript suites -test-all: test-zig test-forth test-ffi test-property test-fuzz-quick - -# ============================================================ -# BENCHMARKS -# ============================================================ - -# Run Factor GQL benchmarks -bench-factor: - @echo "=== Factor GQL Benchmarks ===" - cd core-factor/gql && factor benchmarks.factor - -# Run all benchmarks -bench: bench-factor - -# ============================================================ -# CHECKS & VALIDATION -# ============================================================ - -# Check that Forth code loads without errors -check-forth: - @echo "Checking Forth code loads..." - cd core-forth/src && gforth lithoglyph-blocks.fs -e 'bye' - cd core-forth/src && gforth lithoglyph-journal.fs -e 'bye' - cd core-forth/src && gforth lithoglyph-model.fs -e 'bye' - @echo "All Forth files load successfully" - -# Check that Lean code compiles -check-lean: - @echo "Checking Lean code compiles..." - cd normalizer/lean && lake build - @echo "Lean code compiles" - -# Verify Zig ABI exports match expectations -check-abi: build-zig - @echo "=== ABI Export Verification ===" - nm -D core-zig/zig-out/lib/liblith_bridge.so | grep ' T lith_' | sort - @echo "Expected: lith_version, lith_db_open, lith_db_close, lith_txn_begin, lith_txn_commit, lith_txn_abort, lith_apply, lith_introspect_schema, lith_render_block, lith_render_journal, lith_blob_free" - -# Run all checks -check: check-forth check-lean check-abi - -# Format Zig code -fmt: - cd core-zig && zig fmt src/ - -# ============================================================ -# CLEAN -# ============================================================ - -# Clean build artifacts -clean: - @echo "Cleaning build artifacts..." - rm -rf core-zig/zig-out core-zig/.zig-cache - rm -f core-zig/test-*.lgh - @echo "Clean complete" - -# ============================================================ -# DEV TOOLS -# ============================================================ - -# Show development environment status -env-status: - @echo "Development Environment Status:" - @echo "================================" - @which zig 2>/dev/null && echo "Zig: $(zig version)" || echo "Zig: NOT INSTALLED" - @which gforth 2>/dev/null && echo "Forth: $(gforth --version 2>&1 | head -1)" || echo "gforth: NOT INSTALLED" - @which factor 2>/dev/null && echo "Factor: installed" || echo "Factor: NOT INSTALLED" - @which lean 2>/dev/null && echo "Lean: $(lean --version 2>&1 | head -1)" || echo "Lean: NOT INSTALLED" - @which deno 2>/dev/null && echo "Deno: $(deno --version 2>&1 | head -1)" || echo "Deno: NOT INSTALLED" - -# Start demo server for E2E testing -serve: - @echo "Starting Lithoglyph demo server on :8080..." - cd core-zig && zig run ../demo-server.zig -- -lc diff --git a/lithoglyph/LICENSE b/lithoglyph/LICENSE deleted file mode 100644 index ec540b34..00000000 --- a/lithoglyph/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: MPL-2.0 -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/lithoglyph/MAINTAINERS.adoc b/lithoglyph/MAINTAINERS.adoc deleted file mode 100644 index 48d97817..00000000 --- a/lithoglyph/MAINTAINERS.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Maintainers -:toc: preamble - -This document lists the maintainers of this project and their responsibilities. - -== Current Maintainers - -[cols="2,3,2",options="header"] -|=== -| Name | Role | Contact - -| Jonathan D.A. Jewell -| Lead Maintainer -| https://github.com/hyperpolymath[@hyperpolymath] -|=== - -== Responsibilities - -Maintainers are responsible for: - -* Reviewing and merging pull requests -* Triaging issues and feature requests -* Ensuring code quality and security standards -* Managing releases and versioning -* Upholding the project's code of conduct - -== Becoming a Maintainer - -Contributors who demonstrate: - -* Consistent, high-quality contributions -* Understanding of the project's goals and standards -* Constructive participation in discussions -* Commitment to the project's long-term health - -May be invited to become maintainers at the discretion of existing maintainers. - -== Decision Making - -* Routine decisions (bug fixes, minor improvements) can be made by any maintainer -* Significant changes require discussion and consensus among maintainers -* Breaking changes or major features should be discussed in issues before implementation - -== Contact - -For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/lithoglyph/MONOREPO.md b/lithoglyph/MONOREPO.md deleted file mode 100644 index 8927397d..00000000 --- a/lithoglyph/MONOREPO.md +++ /dev/null @@ -1,24 +0,0 @@ -# Lithoglyph Monorepo Structure - -This repository now contains multiple Lithoglyph components in a monorepo structure: - -## Core Database -- `core-forth/` - Forth storage layer (Form.Blocks, Form.Journal, Form.Model) -- `core-zig/` - Zig FFI bridge (Form.Bridge) -- `core-factor/` - Factor runtime (GQL parser/planner/executor) -- `normalizer/` - Normalization engine with Lean 4 proofs - -## Extensions -- `analytics/` - Analytics module (Julia) -- `geo/` - Geospatial extension (Rust + JSON) -- `beam/` - Erlang BEAM integration - -## Related Repositories -- [lithoglyph-studio](https://github.com/hyperpolymath/lithoglyph-studio) - GUI + debugger -- [gql-dt](https://github.com/hyperpolymath/gql-dt) - Dependently-typed query language (Lean) -- [formbase](https://github.com/hyperpolymath/formbase) - Airtable alternative using Lithoglyph - -## Archived (merged into monorepo) -- lithoglyph-analytics → `analytics/` -- lithoglyph-geo → `geo/` -- lithoglyph-beam → `beam/` diff --git a/lithoglyph/MOVED.md b/lithoglyph/MOVED.md deleted file mode 100644 index c81f94fb..00000000 --- a/lithoglyph/MOVED.md +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell -= Lithoglyph — extraction PARTIALLY DONE - -The Lithoglyph database stack still lives *here*, in `nextgen-databases/lithoglyph/`. -Treat this directory as canonical for everything not listed as extracted below. - -== Extracted - -* Query language: `lithoglyph/gql-dt/` -> https://github.com/hyperpolymath/gnpl - (squash-import `102c79e`, PR #4). *The copy here is no longer canonical.* - -== Not extracted (still canonical here) - -* Database core -> https://github.com/hyperpolymath/lithoglyphdb (reserved 2026-06-22; - migration pending) -* Airtable-mode delivery -> https://github.com/hyperpolymath/glyphbase (reserved - 2026-06-22; migration pending) - -== `core-factor/gql/` — superseded, will NOT be migrated - -`RESITE-DATABASES-TO-OWN-REPOS.adoc` originally specified `gnpl` as `gql-dt/` *plus* -`core-factor/gql/` (5 Factor files: `gql.factor`, `lexer-tests.factor`, -`seam-tests.factor`, `storage-backend.factor`, `benchmarks.factor`). Only the first -path was ever moved — `gnpl` contains zero `.factor` files. - -That is now the settled position, not an omission. `core-factor/gql/` was the -runtime/dynamic counterpart to GQL-dt's compile-time proofs. GNPL has since been -re-scoped as a *narration/projection* language that lowers to GQL-dt, which supersedes -the Factor implementation rather than migrating it. The Factor sources stay here as -legacy. - -CAUTION: an earlier version of this file claimed the code had moved to a `lith` -monorepo. That was INCORRECT — no such consolidation was authorised and no `lith` -repo exists. diff --git a/lithoglyph/Mustfile b/lithoglyph/Mustfile deleted file mode 100644 index 5f075413..00000000 --- a/lithoglyph/Mustfile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Mustfile - hyperpolymath mandatory checks -# See: https://github.com/hyperpolymath/mustfile - -version: 1 - -checks: - - name: security - run: just lint - - name: tests - run: just test - - name: format - run: just fmt diff --git a/lithoglyph/PALIMPSEST.adoc b/lithoglyph/PALIMPSEST.adoc deleted file mode 100644 index e9d2df3a..00000000 --- a/lithoglyph/PALIMPSEST.adoc +++ /dev/null @@ -1,41 +0,0 @@ -= Palimpsest License -:toc: -:toc-placement!: - -image:https://img.shields.io/badge/License-MPL--2.0-blue.svg[License: PMPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] -image:https://img.shields.io/badge/Philosophy-Palimpsest-indigo.svg[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] - -toc::[] - -== Legal Status - -This project is licensed under the **Palimpsest-MPL License 1.0 (PMPL-1.0)**. -For SPDX and tooling, use **PMPL-1.0-or-later**. - -PMPL-1.0 incorporates the Mozilla Public License 2.0 by reference and adds -ethical-use, provenance, and lineage requirements. - -== What PMPL Adds - -* **Emotional Lineage** - preserve narrative intent and cultural context -* **Provenance Integrity** - retain attribution and lineage metadata -* **Ethical Use Constraints** - explicit consent for non-interpretive AI training -* **Quantum-Safe Provenance (optional)** - post-quantum signature support - -== How to Adopt - -1. Include the PMPL-1.0 license text in `LICENSE`. -2. Add SPDX headers to source files: - `SPDX-License-Identifier: CC-BY-SA-4.0` -3. Add a Palimpsest badge to your README (see `assets/badges/` and `embed/license-blocks/`). - -== Versioning - -See `VERSIONING.adoc` for the release process and the "-or-later" model. -The current legal text is PMPL-1.0. - -== References - -* `legal/README.adoc` -* `assets/badges/README.md` -* `embed/license-blocks/README.md` diff --git a/lithoglyph/PHILOSOPHY.adoc b/lithoglyph/PHILOSOPHY.adoc deleted file mode 100644 index 7076fcac..00000000 --- a/lithoglyph/PHILOSOPHY.adoc +++ /dev/null @@ -1,145 +0,0 @@ -= Lith Philosophy -:toc: -:icons: font - -[.lead] -The principles and values that guide Lith's design. - -== Core Thesis - -[quote] -Schemas, constraints, migrations, blocks, and journals are narrative artefacts. The database is part of the story, not an opaque substrate. - -Traditional databases hide their internals. Lith exposes them as readable, auditable artefacts. - -== Value Hierarchy - -Lith explicitly prioritizes certain values over others: - -=== Auditability > Performance - -If a performance optimization obscures auditability, we don't make it. - -Every operation should be traceable. Every state change should be explainable. If we can't answer "why is this data here?" then we've failed. - -=== Meaning > Features - -Features that add capability but reduce meaning are rejected. - -A database full of features nobody understands is worse than a simple one everyone can audit. We add features only when they add meaning. - -=== Reversibility > Throughput - -If an optimization makes operations irreversible, we don't make it. - -The ability to understand "how did we get here?" and "what was the state at time T?" is more valuable than faster writes. - -=== Agent Understanding Required - -Human or AI agents must be able to understand the database state. - -This isn't just about documentation. The database itself must be readable. Blocks must render to meaningful text. Journals must tell coherent stories. - -== Design Principles - -=== Truth Core First, Porous Edges - -The truth core (Form.Blocks, Form.Model) is small, simple, and inviolable. Higher layers are convenience, not truth. - -[source,text] ----- - ┌─────────────────┐ - │ Applications │ ← Porous: flexible, adaptable - ├─────────────────┤ - │ Runtime │ ← Porous: language choice - ├─────────────────┤ - │ Bridge │ ← Boundary: stable ABI - ├─────────────────┤ - │ Model │ ← Truth Core: strict invariants - ├─────────────────┤ - │ Blocks │ ← Truth Core: strict invariants - └─────────────────┘ ----- - -=== Journal-First Semantics - -Every mutation is recorded in the journal before it's considered committed. - -This isn't just for durability—it's for meaning. The journal is the authoritative record of what happened and why. - -=== Constraints as Ethics - -Constraints aren't just validation rules. They're explainable ethical boundaries. - -When a constraint rejects an operation, it must explain: -- What rule was violated -- Why that rule exists -- What would need to change for the operation to succeed - -=== Renderability is Non-Negotiable - -Every internal structure must have a deterministic text rendering. - -Not for debugging. For auditing. A human (or AI) must be able to read the database without specialized tools. - -== Anti-Patterns We Avoid - -=== Hidden State - -No global variables. No implicit caches. No magic. - -If state exists, it's in blocks. If a mutation happened, it's in the journal. - -=== Opaque Errors - -"Error code 42" is unacceptable. - -Every error must explain what happened, why it happened, and what to do about it. - -=== Implicit Behavior - -No auto-migrations. No background compaction without journaling. No silent fixes. - -Every action the database takes should be visible and reversible. - -=== Feature Creep - -Lith will never be a drop-in PostgreSQL replacement. - -We serve specific use cases (journalism, compliance, archives, agents) extremely well, rather than serving all use cases poorly. - -== Target Users - -=== Investigative Journalists - -Who need to prove data provenance and maintain audit trails for legal protection. - -=== Compliance Officers - -Who need to answer "what was the state at time T?" with legal certainty. - -=== Agentic Systems - -Where AI agents need to understand and reason about data state, not just query it. - -=== Cultural Archivists - -Who measure success in decades, not milliseconds. - -== What Lith Is Not - -* A performance-optimized OLTP database -* A SQL database with extra features -* A general-purpose data store -* A distributed system (yet) - -== Conclusion - -Lith exists because we believe: - -1. Databases should be readable -2. History should be preserved -3. Constraints should explain themselves -4. The database is part of the story - -If these beliefs resonate with you, welcome. If they don't, there are many excellent databases that prioritize other values. diff --git a/lithoglyph/PROPOSED-MONOREPO-STRUCTURE.md b/lithoglyph/PROPOSED-MONOREPO-STRUCTURE.md deleted file mode 100644 index 3b3ee83a..00000000 --- a/lithoglyph/PROPOSED-MONOREPO-STRUCTURE.md +++ /dev/null @@ -1,95 +0,0 @@ -# Proposed Lithoglyph Monorepo Structure - -## Current State: 30 top-level directories (too many!) - -## Proposed Clean Structure: - -``` -lithoglyph/ -├── core/ # Core database engine -│ ├── forth/ # Form.Blocks, Form.Journal, Form.Model -│ ├── zig/ # Form.Bridge (FFI layer) -│ └── factor/ # Form.Runtime (GQL parser/executor) -│ -├── extensions/ # Optional extensions -│ ├── analytics/ # Analytics module (Julia) -│ ├── geo/ # Geospatial extension (Rust) -│ └── beam/ # BEAM integration (Erlang) -│ -├── normalizer/ # Self-normalizing engine -│ ├── lean/ # Lean 4 proofs -│ └── factor/ # FD discovery algorithms -│ -├── api/ # HTTP/gRPC API servers -│ ├── http/ # REST API -│ └── grpc/ # gRPC service -│ -├── control-plane/ # Elixir/OTP control plane (optional) -│ -├── clients/ # Client libraries -│ ├── rust/ -│ ├── javascript/ -│ └── python/ -│ -├── integrations/ # Third-party integrations -│ ├── postgres/ -│ ├── kafka/ -│ └── elasticsearch/ -│ -├── tools/ # CLI tools and utilities -│ ├── cli/ # Main CLI tool -│ ├── inspector/ # Database inspector -│ └── migrator/ # Migration tool -│ -├── spec/ # Specifications -│ ├── blocks.adoc -│ ├── journal.adoc -│ ├── gql.adoc -│ └── self-normalizing.adoc -│ -├── docs/ # Documentation -│ ├── guides/ -│ ├── api-reference/ -│ └── architecture/ -│ -├── examples/ # Example code and demos -│ -├── test-vectors/ # Golden test vectors -│ -└── .infrastructure/ # Repo infrastructure - ├── ai-cli-crash-capture/ - ├── contractiles/ - └── licenses/ -``` - -## Cleanup Actions: - -1. **Merge duplicate core directories**: - - `core-forth/` → `core/forth/` - - `core-zig/` → `core/zig/` - - `core-factor/` → `core/factor/` - -2. **Delete old artifacts**: - - `lith/` (old name) - - `lith/` (old name) - - `build/` (should be in .gitignore) - - `ffi/` (duplicate of core/zig?) - -3. **Reorganize extensions**: - - `analytics/`, `geo/`, `beam/` → `extensions/` - -4. **Move infrastructure**: - - `ai-cli-crash-capture/` → `.infrastructure/` - - `contractiles/` → `.infrastructure/` - - `licenses/` → `.infrastructure/` - -5. **Consolidate distributed**: - - `distributed/` → `control-plane/distributed/` (if related) - - OR keep separate if it's a distributed consensus system - -## Benefits: - -- **Clearer navigation**: 10-12 top-level dirs instead of 30 -- **Logical grouping**: Core vs extensions vs tools -- **Easier to find things**: Everything has a clear place -- **Better for newcomers**: Can understand structure at a glance diff --git a/lithoglyph/QUICKSTART.adoc b/lithoglyph/QUICKSTART.adoc deleted file mode 100644 index 7396e076..00000000 --- a/lithoglyph/QUICKSTART.adoc +++ /dev/null @@ -1,696 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Quickstart -:toc: macro -:toc-title: Contents -:toclevels: 2 -:icons: font -:source-highlighter: rouge - -[.lead] -Get Lith running and understand its core concepts in 15 minutes. - -[NOTE] -==== -**Project Status: Proof of Concept** - -Lith is in active development. The core Forth implementation (Form.Blocks, Form.Model) is functional. The GQL runtime and higher layers are being built. This quickstart demonstrates the intended workflow. -==== - -toc::[] - -== What is Lith? - -Lith is a *narrative-first, reversible, audit-grade database*. Unlike traditional databases that treat history as an afterthought, Lith makes provenance, reversibility, and explainability core features. - -[cols="1,2"] -|=== -| If you need... | Lith provides... - -| Audit trails -| Built-in journal with full operation history - -| Data provenance -| Every mutation records who, when, and why - -| Undo/redo -| Every operation has a defined inverse - -| Explainable constraints -| Rejections include reasons and suggestions - -| Graph relationships -| First-class edges with traversal -|=== - -**Target domains**: Investigative journalism, governance/compliance, agentic AI ecosystems, long-term archives. - -== Prerequisites - -=== Required Tools - -[cols="1,1,2"] -|=== -| Tool | Version | Purpose - -| https://github.com/cisco/gforth[Gforth] -| 0.7.3+ -| Form.Blocks and Form.Model (truth core) - -| https://factorcode.org/[Factor] -| 0.99+ -| Form.Runtime (GQL parser/executor) - -| https://ziglang.org/[Zig] -| 0.11+ -| Form.Bridge (stable ABI for FFI) - -| https://just.systems/[just] -| 1.0+ -| Build orchestration -|=== - -=== Installation - -.Fedora/RHEL -[source,bash] ----- -sudo dnf install gforth -curl -LO https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz -tar xf zig-linux-x86_64-0.11.0.tar.xz -sudo mv zig-linux-x86_64-0.11.0 /opt/zig -echo 'export PATH=$PATH:/opt/zig' >> ~/.bashrc -# Factor: download from https://factorcode.org/ ----- - -.Ubuntu/Debian -[source,bash] ----- -sudo apt install gforth -# Zig and Factor: download from official sites ----- - -.macOS -[source,bash] ----- -brew install gforth zig just -# Factor: download from https://factorcode.org/ ----- - -.Arch Linux -[source,bash] ----- -sudo pacman -S gforth zig just -# Factor: AUR or download from factorcode.org ----- - -== Clone and Build - -[source,bash] ----- -git clone https://github.com/hyperpolymath/lithoglyph.git -cd lith - -# Verify Gforth works with our Forth code -gforth core-forth/src/lithoglyph-blocks.fs -e "cr .\" Form.Blocks loaded successfully\" cr bye" ----- - -Expected output: ----- -Form.Blocks loaded successfully ----- - -== Understanding the Architecture - -Lith is structured in layers: - -[source,text] ----- -┌─────────────────────────────────────────────────────────────┐ -│ Form.ControlPlane (Elixir/OTP - optional) │ -│ Sessions, supervision, cluster coordination │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Normalizer (Factor + Lean 4) │ -│ FD discovery, type encoding, proof-carrying normalization │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Runtime (Factor) │ -│ GQL parse/plan/exec, explain, introspection │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Bridge (Zig) │ -│ Stable ABI (no C dependency), safety governor │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Model (Forth) │ -│ Documents, edges, schemas, constraints │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Blocks (Forth) ◀── TRUTH CORE │ -│ 4 KiB blocks, append-only journal, integrity checks │ -└─────────────────────────────────────────────────────────────┘ ----- - -The Forth layers (Form.Blocks, Form.Model) own the truth. Higher layers cannot bypass them. - -== Core Concepts - -=== 1. The Journal (Append-Only Truth) - -Every mutation is journaled *before* being considered committed: - -[source,text] ----- -Journal Entry #1 -├── sequence: 1 -├── op_type: COLLECTION_CREATE -├── timestamp: 2026-01-12T10:00:00Z -├── actor: "alice" -├── rationale: "Setting up evidence tracking for Wirecard investigation" -└── inverse: DROP COLLECTION evidence - -Journal Entry #2 -├── sequence: 2 -├── op_type: DOC_INSERT -├── collection: "evidence" -├── doc_id: "ev-001" -├── actor: "alice" -├── rationale: "Primary source from FT whistleblower" -└── inverse: DELETE FROM evidence WHERE _id = "ev-001" ----- - -This means you can: - -* Replay the journal to any point in time -* Audit who did what and why -* Undo any operation by applying its inverse - -=== 2. Provenance (Who, When, Why) - -Every mutation requires provenance: - -[source,gql] ----- -INSERT INTO evidence { - "title": "Wirecard Internal Audit Report", - "source": "Whistleblower X", - "prompt_score": 85, - "classification": "confidential" -} -WITH PROVENANCE { - "actor": "dan.mccrum", - "rationale": "Received via secure drop. Cross-referenced with public filings.", - "source_type": "primary", - "verification_method": "document_hash_verified" -}; ----- - -The `actor` and `rationale` fields are required. Additional provenance fields are optional but encouraged. - -=== 3. PROMPT Scores (Evidence Quality) - -Lith includes a built-in `PROMPT_SCORE` type (0-100) for rating evidence quality: - -[cols="1,2"] -|=== -| Score Range | Meaning - -| 90-100 | Primary source, verified, official -| 70-89 | Secondary source, corroborated -| 50-69 | Single source, unverified -| 30-49 | Rumor, speculation, needs verification -| 0-29 | Dubious, contradicted, discredited -|=== - -[source,gql] ----- --- Find all high-confidence evidence -SELECT * FROM evidence WHERE prompt_score >= 80; - --- Find evidence that needs verification -SELECT * FROM evidence WHERE prompt_score BETWEEN 30 AND 69; ----- - -=== 4. Edges (Graph Relationships) - -Lith supports edges as first-class citizens: - -[source,gql] ----- --- Create an edge connecting a claim to its evidence -INSERT EDGE supports FROM claims/claim-001 TO evidence/ev-042 { - "strength": "strong", - "relationship": "directly_supports" -} -WITH PROVENANCE { - "actor": "researcher_bob", - "rationale": "Document explicitly confirms the claim" -}; ----- - -Traverse edges with GQL: - -[source,gql] ----- --- Find all evidence supporting a claim -SELECT e.* -FROM claims -WHERE _id = "claim-001" -TRAVERSE supports OUTBOUND -DEPTH 1; - --- Find what claims a piece of evidence supports -SELECT c.* -FROM evidence -WHERE _id = "ev-042" -TRAVERSE supports INBOUND; - --- Find the full evidence chain (multi-hop) -SELECT * -FROM claims -WHERE _id = "claim-001" -TRAVERSE supports OUTBOUND -DEPTH 3; ----- - -=== 5. Reversibility - -Every operation has a defined inverse: - -[source,gql] ----- --- See what undoing the last operation would do -EXPLAIN REVERSE LAST; - --- Actually reverse the last operation -REVERSE LAST WITH PROVENANCE { - "actor": "editor_carol", - "rationale": "Document was misattributed - correct source is SEC filing" -}; - --- Reverse to a specific journal sequence -REVERSE TO SEQUENCE 42 WITH PROVENANCE { - "actor": "admin_dave", - "rationale": "Rolling back corrupted batch import" -}; ----- - -Some operations are *irreversible-with-story*: they can't be undone, but the journal still records what happened and why. - -=== 6. Explainable Constraints - -When constraints fail, Lith explains why: - -[source,gql] ----- -INSERT INTO evidence (source) VALUES ('Anonymous tip'); ----- - -Error response: - -[source,json] ----- -{ - "status": "error", - "code": "CONSTRAINT_VIOLATION", - "constraint": "title_required", - "message": "The 'title' field is required for all evidence records", - "field": "title", - "context": { - "collection": "evidence", - "schema_version": 1, - "constraint_definition": { - "type": "not_null", - "field": "title", - "rationale": "All evidence must be identifiable for audit purposes" - } - }, - "suggestions": [ - "Provide a descriptive title for this evidence", - "If the source is anonymous, title could be 'Anonymous tip re: [topic]'", - "Use INTROSPECT SCHEMA evidence to see all required fields" - ], - "journal_sequence": null, - "operation_applied": false -} ----- - -== Tutorial: Building an Evidence Database - -This tutorial walks through building a simple evidence database for investigative journalism. - -=== Step 1: Create Collections - -[source,gql] ----- --- Evidence collection with quality scoring -CREATE COLLECTION evidence ( - title STRING NOT NULL, - description STRING, - source STRING NOT NULL, - source_type STRING CHECK (source_type IN ('primary', 'secondary', 'tertiary')), - prompt_score PROMPT_SCORE NOT NULL, - url STRING, - document_hash STRING, - classification STRING CHECK (classification IN ('public', 'confidential', 'restricted')), - tags JSON, - obtained_date TIMESTAMP -) WITH SCHEMA { - "description": "Evidence artifacts for investigative stories", - "constraints": { - "title_required": { - "type": "not_null", - "field": "title", - "rationale": "All evidence must be identifiable" - }, - "source_required": { - "type": "not_null", - "field": "source", - "rationale": "Provenance requires knowing the source" - }, - "quality_required": { - "type": "not_null", - "field": "prompt_score", - "rationale": "All evidence must be rated for reliability" - } - } -}; - --- Claims that evidence supports or refutes -CREATE COLLECTION claims ( - statement STRING NOT NULL, - subject STRING NOT NULL, - status STRING CHECK (status IN ('unverified', 'corroborated', 'disputed', 'refuted', 'confirmed')), - confidence FLOAT CHECK (confidence >= 0.0 AND confidence <= 1.0), - first_reported TIMESTAMP, - public BOOLEAN -); - --- Sources (people, organizations, documents) -CREATE COLLECTION sources ( - name STRING NOT NULL, - type STRING CHECK (type IN ('person', 'organization', 'document', 'dataset')), - reliability_score PROMPT_SCORE, - notes STRING, - contact_info JSON -); ----- - -=== Step 2: Insert Evidence - -[source,gql] ----- --- Primary source with high confidence -INSERT INTO evidence { - "title": "Annual Financial Statement 2023", - "description": "Audited financial statements showing revenue discrepancy", - "source": "Company SEC Filing", - "source_type": "primary", - "prompt_score": 95, - "url": "https://sec.gov/cgi-bin/browse-edgar?action=getcompany&...", - "document_hash": "sha256:abc123...", - "classification": "public", - "tags": ["financial", "annual-report", "2023"], - "obtained_date": "2024-03-15T09:00:00Z" -} -WITH PROVENANCE { - "actor": "alice.journalist", - "rationale": "Official SEC filing, publicly available, verified hash", - "verification": "Downloaded directly from SEC EDGAR" -}; - --- Secondary source with medium confidence -INSERT INTO evidence { - "title": "Internal Email Thread: Budget Concerns", - "description": "Leaked email chain discussing budget irregularities", - "source": "Anonymous whistleblower", - "source_type": "secondary", - "prompt_score": 65, - "classification": "confidential", - "tags": ["email", "internal", "budget"], - "obtained_date": "2024-04-02T14:30:00Z" -} -WITH PROVENANCE { - "actor": "bob.researcher", - "rationale": "Received via SecureDrop. Headers appear genuine but not independently verified.", - "verification": "DKIM signature present, awaiting independent confirmation" -}; ----- - -=== Step 3: Create Claims and Link to Evidence - -[source,gql] ----- --- Create a claim -INSERT INTO claims { - "statement": "Company X overstated 2023 revenue by $2.3 billion", - "subject": "Company X", - "status": "corroborated", - "confidence": 0.85, - "first_reported": "2024-04-10T00:00:00Z", - "public": false -} -WITH PROVENANCE { - "actor": "alice.journalist", - "rationale": "Claim derived from SEC filing analysis and whistleblower documents" -}; - --- Link evidence to claim with edges -INSERT EDGE supports FROM claims/claim-001 TO evidence/ev-001 { - "strength": "strong", - "notes": "SEC filing shows $2.1B discrepancy in Note 7" -} -WITH PROVENANCE { - "actor": "alice.journalist", - "rationale": "Direct numerical evidence from audited financials" -}; - -INSERT EDGE supports FROM claims/claim-001 TO evidence/ev-002 { - "strength": "moderate", - "notes": "Email discusses 'booking revenue early' - circumstantial" -} -WITH PROVENANCE { - "actor": "bob.researcher", - "rationale": "Corroborates but doesn't prove - adds context" -}; ----- - -=== Step 4: Query and Analyze - -[source,gql] ----- --- Find all high-quality evidence -SELECT title, source, prompt_score -FROM evidence -WHERE prompt_score >= 80 -ORDER BY prompt_score DESC; - --- Find all evidence for a specific claim with provenance -SELECT e.*, _provenance -FROM claims c -WHERE c._id = "claim-001" -TRAVERSE supports OUTBOUND -WITH PROVENANCE; - --- Explain a query -EXPLAIN -SELECT * -FROM evidence -WHERE source_type = "primary" AND prompt_score >= 90; ----- - -The `EXPLAIN` output shows: - -[source,json] ----- -{ - "plan": { - "type": "SCAN", - "collection": "evidence", - "filters": [ - {"field": "source_type", "op": "=", "value": "primary"}, - {"field": "prompt_score", "op": ">=", "value": 90} - ], - "estimated_cost": 0.3, - "index_used": null - }, - "constraints_checked": ["source_type_valid", "prompt_score_range"], - "provenance_included": false, - "rationale": "Full collection scan with dual filter. Consider adding index on (source_type, prompt_score) for frequent queries." -} ----- - -=== Step 5: Introspect the Database - -[source,gql] ----- --- List all collections -INTROSPECT COLLECTIONS; - --- View schema for evidence collection -INTROSPECT SCHEMA evidence; - --- View all constraints -INTROSPECT CONSTRAINTS evidence; - --- View recent journal entries -INTROSPECT JOURNAL SINCE 0 LIMIT 20; - --- View journal for specific collection -INTROSPECT JOURNAL -WHERE collection = "evidence" -SINCE 0; ----- - -=== Step 6: Undo a Mistake - -[source,gql] ----- --- Oops, we inserted evidence with wrong source --- First, see what would be undone -EXPLAIN REVERSE LAST; - --- Reverse it -REVERSE LAST WITH PROVENANCE { - "actor": "bob.researcher", - "rationale": "Source was incorrectly attributed. Correct source is 'Financial Times investigation'" -}; - --- Re-insert with correct information -INSERT INTO evidence { - "title": "Internal Email Thread: Budget Concerns", - "source": "Financial Times investigation", - "source_type": "secondary", - "prompt_score": 75, -- Upgraded confidence after verification - ... -} -WITH PROVENANCE { - "actor": "bob.researcher", - "rationale": "Corrected source attribution. FT independently verified email authenticity." -}; ----- - -== Block Storage Details - -For those interested in the internals, Lith uses 4 KiB fixed-size blocks: - -[source,text] ----- -Block Header (64 bytes) -├── magic (4 bytes): "LGH\0" (0x4C474800) -├── version (2 bytes): Format version -├── block_type (2 bytes): TYPE-DOCUMENT, TYPE-EDGE, etc. -├── block_id (8 bytes): Unique block identifier -├── sequence (8 bytes): Journal sequence number -├── created_at (8 bytes): Timestamp -├── modified_at (8 bytes): Timestamp -├── payload_len (4 bytes): Actual payload size -├── checksum (4 bytes): CRC32C of payload -├── prev_block_id (8 bytes): For chained blocks -├── flags (4 bytes): COMPRESSED, ENCRYPTED, CHAINED, DELETED -└── reserved (4 bytes): Future use - -Block Payload (4032 bytes) -└── [Document/Edge/Schema data] ----- - -Block types include: - -[cols="1,1,2"] -|=== -| Type | Code | Purpose - -| TYPE-SUPERBLOCK | 0x0001 | Database metadata -| TYPE-COLLECTION-META | 0x0010 | Collection schema -| TYPE-DOCUMENT | 0x0011 | Document data -| TYPE-EDGE | 0x0021 | Edge data -| TYPE-JOURNAL-SEGMENT | 0x0040 | Journal entries -| TYPE-SCHEMA | 0x0050 | Schema definitions -| TYPE-CONSTRAINT | 0x0051 | Constraint definitions -| TYPE-MIGRATION | 0x0060 | Migration artefacts -|=== - -== Next Steps - -* link:spec/gql.adoc[GQL Language Reference] - Complete query language specification -* link:spec/gql-dependent-types.md[FQLdt Specification] - Compile-time verified queries with Lean 4 -* link:ARCHITECTURE.adoc[Architecture Guide] - Deep dive into the layer design -* link:docs/DEPLOYMENT.adoc[Deployment Guide] - Run Lith in production -* link:spec/self-normalizing.adoc[Self-Normalizing Spec] - Automatic functional dependency discovery - -== Troubleshooting - -=== "gforth: command not found" - -Install Gforth: - -[source,bash] ----- -# Fedora/RHEL -sudo dnf install gforth - -# Ubuntu/Debian -sudo apt install gforth - -# macOS -brew install gforth - -# Arch Linux -sudo pacman -S gforth ----- - -=== "Error: BLOCKS word redefined" - -The Forth code resets the vocabulary to avoid conflicts with Gforth's built-in BLOCKS extension. If you still see conflicts, ensure you're using a clean Gforth session: - -[source,bash] ----- -gforth -e "only forth definitions" core-forth/src/lithoglyph-blocks.fs ----- - -=== "Permission denied on journal" - -Lith requires write access to the database directory: - -[source,bash] ----- -chmod -R u+w mydb/ ----- - -=== "Constraint violation on valid data" - -Inspect the schema to understand constraints: - -[source,gql] ----- -INTROSPECT SCHEMA evidence; -INTROSPECT CONSTRAINTS evidence; ----- - -The error message includes `suggestions` - read them carefully. - -=== "Provenance required" - -All mutations require a `WITH PROVENANCE` clause: - -[source,gql] ----- --- Wrong -INSERT INTO evidence {...}; - --- Correct -INSERT INTO evidence {...} -WITH PROVENANCE { - "actor": "your_username", - "rationale": "Why you're making this change" -}; ----- - -== Getting Help - -* https://github.com/hyperpolymath/lithoglyph/issues[GitHub Issues] - Report bugs, request features -* https://github.com/hyperpolymath/lithoglyph/discussions[GitHub Discussions] - Ask questions -* link:CONTRIBUTING.adoc[Contributing Guide] - Join development -* link:ROADMAP.adoc[Roadmap] - See what's planned - -== Related Projects - -* https://github.com/hyperpolymath/gqldt[GQLdt] - Dependently-typed GQL (compile-time verified queries) -* https://github.com/hyperpolymath/lithoglyph-studio[Lith Studio] - Zero-friction GUI -* https://github.com/hyperpolymath/lithoglyph-debugger[Lith Debugger] - Proof-carrying debugger -* https://github.com/hyperpolymath/formbase[FormBase] - Open-source Airtable alternative with provenance diff --git a/lithoglyph/README.adoc b/lithoglyph/README.adoc index 79f608b4..f380ef65 100644 --- a/lithoglyph/README.adoc +++ b/lithoglyph/README.adoc @@ -1,250 +1,60 @@ -= Lithoglyph - -image:https://img.shields.io/badge/License-MPL--2.0-indigo.svg[License: PMPL-1.0-or-later,link="https://github.com/hyperpolymath/palimpsest-license"] // SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2025-2026 hyperpolymath - -:toc: macro -:toc-title: Contents -:toclevels: 3 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += Lithoglyph — extracted :icons: font -:source-highlighter: rouge - -image:https://img.shields.io/badge/standard-RSR--compliant-green.svg[RSR Compliant] - -[.lead] -*Stone-carved data for the ages.* - -_Where Forth carves glyphs into permanence._ - -toc::[] - -== Overview - -[NOTE] -==== -**Project status: Active, secondary priority.** LithoGlyph is working towards production for specialist use in **journalism, narrative arts, and media/communications**. It is not the primary priority project right now — that is https://github.com/hyperpolymath/verisimdb[VeriSimDB], which is a substantial undertaking in cross-modal database consistency. Lessons learned from VeriSimDB (drift detection, formal verification, federation architecture) will directly improve LithoGlyph when development focus returns here. -**For most journalism, storytelling, and media workflows, LithoGlyph and its query language GQL are likely the database you want.** The narrative-first, reversible, audit-grade design is built specifically for this domain. https://github.com/hyperpolymath/verisimdb[VeriSimDB] and https://github.com/hyperpolymath/gql-dt[VCL/VCL-UT] become more relevant when your work involves cross-source investigative verification, large-scale entity tracking across multiple systems, long-term identity consistency, or situations where you need to prove that data across different databases has not drifted apart. For everything else — the story you want to write, the sources you want to cite, the narrative you want to build — LithoGlyph is designed to be your storage layer. +Lithoglyph no longer lives here. It is developed at: -In the long run, LithoGlyph is intended to work intimately with tools like bofig in the portfolio and serve as a database for journalism and PR applications more broadly. -==== +https://github.com/hyperpolymath/lithoglyph[hyperpolymath/lithoglyph] -**Lithoglyph** (from Greek *lithos* "stone" + *glyphein* "to carve") is a *narrative-first, reversible, audit-grade database core*. Like ancient glyphs carved in stone, Lithoglyph treats schemas, constraints, migrations, blocks, and journals as permanent narrative artefacts—the database is part of the story, not an opaque substrate. +The 819 files that used to sit under this directory were removed on 2026-07-27, +after the extraction they were waiting on completed +(`hyperpolymath/lithoglyph#4`, merged). Keeping them here left the whole database +in two places with no canonical copy — the defect the extraction existed to fix. -The name reflects the core metaphor: **Forth code sculpts data onto disk**, carving each operation into permanent, auditable stone. +== Where each part went -=== Core Thesis +[cols="2,3"] +|=== +| Was | Now -> Schemas, constraints, migrations, blocks, and journals are narrative artefacts. -> The database is part of the story, not an opaque substrate. -> Data is carved in stone—permanent, auditable, reversible. +| `lithoglyph/` (the core) +| https://github.com/hyperpolymath/lithoglyph[hyperpolymath/lithoglyph] -=== Primary Values +| `lithoglyph/glyphbase/` (130 files) +| https://github.com/hyperpolymath/glyphbase[hyperpolymath/glyphbase] -[cols="2,1"] -|=== -| Principle | Priority +| `lithoglyph/gql-dt/` (125 files) +| https://github.com/hyperpolymath/gnpl[hyperpolymath/gnpl] -| Auditability | > Performance -| Meaning | > Features -| Reversibility | > Throughput -| Agent Understanding | Required +| `lithoglyph/core-factor/gql/`, `gql-c-api/` +| Superseded, not migrated — see below |=== -== Target Domains +`core-factor/gql/` was the runtime counterpart to GQLdt's compile-time proofs. A +typed narration layer lowering to GQLdt replaces it, so it was never migrated and +should not be "finished". See `docs/migration/RESITE-DATABASES-TO-OWN-REPOS.adoc` +and the GNPL re-scope recorded in #56. -* Investigative journalism -* Governance/compliance -* Agentic ecosystems + multi-repo handover -* Long-term cultural/institutional archives +== Nothing was lost -== Repository Structure +Every file removed here is preserved at its original path in the annotated tag +`split-history/lithoglyph`, which is on `origin` and holds all 819 files across +68 commits of granular pre-squash history: -[source,text] +[source,console] ---- -lithoglyph/ -├── spec/ # Format specs + rationale (AsciiDoc) -├── core-forth/ # Form.Blocks + Form.Model (truth core) -├── core-zig/ # Form.Bridge (ABI + port framing) -├── core-factor/ # Form.Runtime (GQL + introspection) -├── control-plane/ # Elixir/OTP gateway (optional) -├── tools/ # Render/inspect/doctor utilities -├── test-vectors/ # Golden bytes + golden renders -└── stories/ # Narrative examples + onboarding artefacts +$ git ls-tree -r split-history/lithoglyph --name-only | wc -l +819 ---- -== Architecture +To recover any file: -Lithoglyph is structured in layers, each with a specific language and responsibility: - -[source,text] +[source,console] ---- -┌─────────────────────────────────────────────────────────────┐ -│ Form.ControlPlane (Elixir/OTP - optional) │ -│ Sessions, supervision, cluster edge │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Normalizer (Factor + Lean 4) │ -│ FD discovery, type encoding, proof-carrying normalization │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Runtime (Factor) │ -│ GQL parse/plan/exec, explain, introspection │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Bridge (Zig) │ -│ Stable Zig ABI (no C), safety governor │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Model (Forth) │ -│ Multi-model logical layer: documents, edges, schemas │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Blocks (Forth) │ -│ Deterministic storage, journal, reversibility primitives │ -└─────────────────────────────────────────────────────────────┘ +git checkout split-history/lithoglyph -- ---- -=== Layer Details - -==== Form.Blocks (Forth) -* Fixed-size blocks with symbolic headers -* Append-only journal -* Crash recovery and integrity checks -* Repair guidance - -==== Form.Model (Forth) -* Document collections -* Edge collections -* Schema + constraint metadata -* Migration artefacts - -==== Form.Bridge (Zig) -* Stable Zig ABI for runtimes (no C dependency) -* Opaque handles + byte buffers + explicit error codes -* Marshalling only; no business logic duplication -* Uses `callconv(.C)` for FFI compatibility without requiring C toolchain - -==== Form.Runtime (Factor) -* GQL minimal subset for PoC -* Planner steps introspection -* Constraint explanation surfaces -* Provenance surfaces - -==== Form.Normalizer (Factor + Lean 4) -* Automatic functional dependency discovery (DFD/TANE/FDHits) -* Type encoding of FDs in GQL-DT -* Normal form predicates (1NF through BCNF) -* Proof-carrying schema evolution -* Narrative explanations for all normalization decisions -* See link:spec/self-normalizing.adoc[Self-Normalizing Specification] - -==== Form.ControlPlane (Elixir/OTP, optional) -* Out-of-process core engine via port -* Sessions and supervision -* Cluster edge coordination - -== Core Invariants - -These are *non-negotiable* guarantees: - -[horizontal] -Truth Ownership:: On-disk truth is owned by the block/journal layer; higher layers do not bypass it. -Journal-First:: Every mutating operation is journaled before being considered committed. -Reversibility:: Every committed operation MUST have a defined inverse OR be explicitly marked irreversible-with-story. -Renderability:: Blocks and journal entries MUST be renderable deterministically into human/agent-readable form. -Provenance:: All query results can optionally include provenance pointers to journal and blocks. -Constraints as Ethics:: Constraints are explainable: rejections must return reasons + pointers + narrative rationale. - -== GQL (Glyph Query Language) - -GQL is the query language for Lithoglyph—the native query interface for this narrative-first database. The name "Glyph Query Language" reflects the stone-carving metaphor: each query carves new glyphs (data) into permanence. - -PoC subset capabilities: - -* `INSERT` document into collection -* `INSERT` edge (from, to, type, props) -* `SELECT` with simple predicates -* `EXPLAIN` (returns plan + reasons) -* `INTROSPECT` schema/constraints -* Optional provenance output - -== Non-Goals - -* Be a drop-in Postgres replacement -* Win microbenchmarks -* Ship full distributed consensus in the first PoC - -== PoC Acceptance Criteria - -* Single-node db open/close -* Append-only journal with deterministic rendering -* Document + edge insert/select -* Constraint rejection returns explain payload -* Migration artefact recorded + reversible -* Golden test vectors pass -* Seam checks (B↔M, M↔R, B↔R) pass at freeze - -== Open Questions - -See link:lithoglyph.scm[lithoglyph.scm] for structured open questions with acceptance criteria: - -=== Core PoC -* `Q-BLOCK-HEADER-001`: Minimal block header layout -* `Q-JOURNAL-ENTRY-001`: Minimal journal entry schema -* `Q-ABI-BLOBS-001`: ABI blob encoding choice -* `Q-GQL-POC-001`: GQL PoC grammar -* `Q-CTRL-PLANE-001`: Elixir/OTP introduction timing - -=== Self-Normalizing (Form.Normalizer) -* `Q-NORM-001`: Default FD discovery algorithm (DFD vs TANE vs FDHits) -* `Q-NORM-002`: Approximate FD handling policy -* `Q-NORM-003`: Denormalization support scope -* `Q-NORM-004`: GQL-DT proof integration interface -* `Q-NORM-005`: Query rewriting for affected queries - -== Documentation - -=== Getting Started -* link:QUICKSTART.adoc[Quickstart Guide] - Get running in 5 minutes -* link:VERSIONING.adoc[Versioning & Stability] - API stability guarantees -* link:CHANGELOG.md[Changelog] - Release history - -=== Architecture & Design -* link:ARCHITECTURE.adoc[Architecture] - Technical architecture deep-dive -* link:ROADMAP.adoc[Roadmap] - Development roadmap -* link:PHILOSOPHY.adoc[Philosophy] - Design philosophy and principles -* link:lithoglyph.scm[lithoglyph.scm] - Machine-readable specification - -=== Specifications -* link:spec/gql.adoc[GQL Specification] - Query language reference -* link:spec/gql-dependent-types.md[GQL Dependent Types] - Compile-time verified queries -* link:spec/self-normalizing.adoc[Self-Normalizing Database] - Automatic FD discovery, proof-carrying schema evolution -* link:spec/blocks.adoc[Block Format] - Storage layer specification -* link:spec/journal.adoc[Journal Format] - Audit log specification - -=== Operations -* link:docs/DEPLOYMENT.adoc[Deployment Guide] - Run Lithoglyph in production -* link:docs/OBSERVABILITY.adoc[Observability] - Monitoring, logging, tracing -* link:docs/SECURITY-AUTH.adoc[Security & Auth] - Authentication and hardening - -=== Integration -* link:docs/API-REFERENCE.adoc[API Reference] - Programmatic interfaces -* link:docs/MIGRATION-FROM-RDBMS.adoc[Migration from RDBMS] - Moving from PostgreSQL/MySQL -* link:docs/INTEGRATION-PATTERNS.adoc[Integration Patterns] - Queues, search, analytics - -=== Domain-Specific -* link:docs/WP05_interactive_documentary.md[WP05] - Lithoglyph for Interactive Documentary/Journalism - -== Related Projects - -* https://github.com/hyperpolymath/gql-dt[GQL-DT] - Dependently-typed GQL (same database, compile-time proofs) -* https://github.com/hyperpolymath/lithoglyph-studio[Lithoglyph Studio] - Zero-friction GUI for non-technical users -* https://github.com/hyperpolymath/bofig[BoFIG] - Evidence graph for investigative journalism -* https://github.com/hyperpolymath/zotero-lithoglyph[Zotero-Lithoglyph] - Post-truth reference manager with PROMPT scores -* https://github.com/hyperpolymath/lithoglyph-debugger[Lithoglyph Debugger] - Proof-carrying database debugger (Lean 4 + Idris 2) -* https://github.com/hyperpolymath/formbase[FormBase] - Open-source Airtable alternative with provenance - -== License - -https://github.com/hyperpolymath/palimpsest-license[Palimpsest-MPL 1.0] (PMPL-1.0) - -== Contributing - -See link:CONTRIBUTING.adoc[CONTRIBUTING.adoc] for guidelines. +**Never prune** `split-history/lithoglyph` or the `_split_lithoglyph` branch, nor +their `glyphbase` and `gnpl` equivalents. The imports into the standalone repos +were squash merges, so these refs are the only granular history that exists. diff --git a/lithoglyph/RENAME-SUMMARY.md b/lithoglyph/RENAME-SUMMARY.md deleted file mode 100644 index ed402516..00000000 --- a/lithoglyph/RENAME-SUMMARY.md +++ /dev/null @@ -1,133 +0,0 @@ -# Lith → Lith Renaming Complete - -**Date:** 2026-02-04 -**Reason:** Emphasize **B**idirectional/**D**atabase reversibility - ---- - -## What Changed - -### Repository -- **Old:** `/var$REPOS_DIR/lith` -- **New:** `/var$REPOS_DIR/lith` -- **Symlink:** `~/Documents/hyperpolymath-repos/lith` → `/var$REPOS_DIR/lith` - -### Text Replacements (Throughout Codebase) -- `Lith` → `Lith` -- `lith` → `lith` -- `FORMDB` → `LITH` (completed 2026-03-13) - -**Files affected:** ~48 source files (.lean, .zig, .res, .idr, .md, .toml, .json) - -### File Renames -- `lith_query.zig` → `lith_query.zig` -- `liblith_query_ffi.a` → `liblith_query_ffi.a` (build artifact) -- All documentation references updated - ---- - -## Testing Results - -### ✅ All Tests Passing - -```bash -zig build test # ✓ Unit tests (14 tests) -zig build test-integration # ✓ Integration tests (15 tests) -zig build test-property # ✓ Property-based tests (5 invariants) -``` - -**Total:** 34 tests passing - ---- - -## ECHIDNA Integration - -Successfully integrated ECHIDNA's property-based testing pattern to verify Lith invariants: - -1. **Parser Determinism** - Same query → same result (5 iterations) -2. **Lifecycle Stability** - 100 create/destroy cycles without leaks -3. **Null Safety** - Graceful handling of null pointers -4. **Invalid Input** - Malformed queries don't crash -5. **Memory Bounds** - 1000 iterations without unbounded growth - -**Confidence:** HIGH - 1100+ test iterations, formal invariant verification - ---- - -## What Lith Means - -**Lith** = **Form**al **B**idirectional **D**atabase - -### Key Attributes -- **Bidirectional:** Read and write operations are reversible -- **Provenance:** All changes tracked with rationale (RATIONALE clause) -- **Dependent Types:** Compile-time guarantees via Lean 4 -- **Formal Verification:** Idris2 ABI + ECHIDNA property testing - -### Architecture -``` -Lean 4 Specification - ↓ -Idris2 ABI (Formal Verification) - ↓ -Zig FFI (C-Compatible Implementation) - ↓ -ReScript Bindings (Type-Safe Web API) -``` - ---- - -## Build Status - -### ✅ Complete (94%) -- **M7** Idris2 ABI: 598 lines + formal proofs -- **M8** Zig FFI: 522 lines + 340 lines tests (all passing) -- **M9** ReScript: 934 lines type-safe bindings -- **Property Tests:** 5 ECHIDNA-style invariants - -### ⚠️ Known Issues -- Lean 4 Pipeline.lean: 2 universe polymorphism errors (type-checking only) -- **Impact:** None - Zig FFI implementation works correctly -- **Status:** 33/35 Lean modules compiling - ---- - -## Implementation Status - -### Components - -| Component | Status | Lines | Tests | -|-----------|--------|-------|-------| -| Idris2 ABI | ✅ Complete | 598 | Formal proofs | -| Zig FFI | ✅ Complete | 522 | 29 passing | -| ReScript Bindings | ✅ Complete | 934 | Type-safe | -| Property Tests | ✅ Complete | 120 | 5 invariants | -| Lean Pipeline | ⚠️ Type issues | 296 | Axiomatized | - -**Total:** 2,054 production lines + 340 test lines - ---- - -## Next Steps - -1. ✅ Repository renamed -2. ✅ All text references updated -3. ✅ Build system working -4. ✅ All tests passing -5. 🔄 Update GitHub remote URL (if needed) -6. 🔄 Update documentation references -7. 🔄 Announce rename to users/contributors - ---- - -## References - -- **Security Requirements:** [SECURITY-REQUIREMENTS.md](lith/SECURITY-REQUIREMENTS.md) -- **ABI Documentation:** [query/src/abi/Types.idr](lith/query/src/abi/Types.idr) -- **FFI Tests:** [query/ffi/zig/test/](lith/query/ffi/zig/test/) -- **ReScript API:** [query/bindings/rescript/README.md](lith/query/bindings/rescript/README.md) - ---- - -**Author:** Jonathan D.A. Jewell -**License:** PMPL-1.0-or-later diff --git a/lithoglyph/ROADMAP.adoc b/lithoglyph/ROADMAP.adoc deleted file mode 100644 index 6b7c2d4d..00000000 --- a/lithoglyph/ROADMAP.adoc +++ /dev/null @@ -1,440 +0,0 @@ -= Lithoglyph Roadmap -:toc: -:icons: font - -[.lead] -Development phases from concept to production. - -**Note:** This project was formerly known as "lith" until 2026-02-05, when it was renamed to Lithoglyph to avoid trademark conflicts and better reflect the stone-carving metaphor of Forth sculpting data onto disk. - -== Milestone Summary - -[cols="1,2,1,1"] -|=== -| Milestone | Description | Version | Status - -| M1 | Core Specifications | v0.0.2 | ✅ Complete -| M2-M5 | Forth PoC Implementation | v0.0.2 | ✅ Complete -| M6 | Machine-Readable Artefacts | v0.0.2 | ✅ Complete -| M7 | Complete Documentation Suite | v0.0.3 | ✅ Complete -| M8 | Form.Runtime (GQL Engine) | v0.0.4 | ✅ Complete -| M9 | Form.Normalizer | v0.0.4 | ✅ Complete -| M10 | Production Hardening | v0.0.4 | ✅ Complete -| M11 | Multi-Protocol API Server | v0.0.5 | ✅ Complete -| M12 | Language Bindings (ReScript, PHP) | v0.0.6 | 🚧 Next -| M13 | CMS Integration (WordPress) | v0.0.7 | 📋 Planned -| M14 | Form.ControlPlane (Clustering) | v0.1.0 | 📋 Planned -| M14.5 | Final Naming and Branding | v0.9.0 | 📋 Planned -| M15 | 1.0.0 Release Candidate | v1.0.0-rc | 📋 Planned -|=== - -== Phase 0: Specification ✅ - -*Status*: Complete (v0.0.2) - -*Goal*: Complete specifications for all core components. - -=== Deliverables - -* [x] `lithoglyph.scm` - Unified specification (formerly lithoglyph.scm) -* [x] Repository structure -* [x] `spec/blocks.adoc` - Block format spec (4 KiB blocks, 64-byte headers) -* [x] `spec/journal.adoc` - Journal format spec (append-only, sequence numbers) -* [x] `spec/gql.adoc` - GQL grammar (EBNF, provenance syntax) -* [x] `spec/gql-dependent-types.md` - FQLdt with Lean 4 integration -* [x] `spec/self-normalizing.adoc` - Self-normalizing database spec -* [x] `spec/cloud-storage.adoc` - Object storage integration -* [x] `spec/gql-philosophy.adoc` - Design philosophy - -== Phase 1: Form.Blocks PoC ✅ - -*Status*: Complete (v0.0.2) - -*Goal*: Working storage layer with journal and reversibility. - -=== Deliverables - -* [x] Fixed-size block implementation (`core-forth/src/lithoglyph-blocks.fs`) -* [x] Block header structure with magic, version, type, checksums -* [x] Memory buffer management -* [x] CRC32C implementation (Castagnoli polynomial) - -== Phase 2: Form.Journal PoC ✅ - -*Status*: Complete (v0.0.2) - -*Goal*: Append-only journal with crash recovery. - -=== Deliverables - -* [x] Append-only journal (`core-forth/src/lithoglyph-journal.fs`) -* [x] Sequence numbering -* [x] Operation logging with inverses -* [x] Crash recovery primitives - -== Phase 3: Form.Model PoC ✅ - -*Status*: Complete (v0.0.2) - -*Goal*: Multi-model layer on top of blocks. - -=== Deliverables - -* [x] Document collection support (`core-forth/src/lithoglyph-model.fs`) -* [x] Edge collection support -* [x] Schema metadata storage -* [x] Constraint storage - -== Phase 4: Form.Bridge ✅ - -*Status*: Complete (v0.0.2) - -*Goal*: Stable ABI for runtime integration. - -=== Deliverables - -* [x] Zig-only ABI (C dependency eliminated) -* [x] FFI design for Factor integration - -=== Note - -Architecture changed from C ABI to Zig-only ABI for better safety and integration. - -== Phase 5: Documentation ✅ - -*Status*: Complete (v0.0.3) - -*Goal*: Comprehensive documentation for production use. - -=== Deliverables - -* [x] `QUICKSTART.adoc` - 15-minute tutorial -* [x] `VERSIONING.adoc` - Stability policy -* [x] `docs/DEPLOYMENT.adoc` - Docker, Kubernetes, systemd -* [x] `docs/SECURITY-AUTH.adoc` - Authentication, authorization, hardening -* [x] `docs/API-REFERENCE.adoc` - Form.Bridge FFI, GQL API -* [x] `docs/MIGRATION-FROM-RDBMS.adoc` - PostgreSQL/MySQL/SQLite migration -* [x] `docs/OBSERVABILITY.adoc` - Logging, metrics, tracing, dashboards -* [x] `docs/INTEGRATION-PATTERNS.adoc` - Message queues, search, analytics, AI/ML - -== Phase 6: Form.Runtime PoC ✅ - -*Status*: Complete (v0.0.4) - -*Goal*: GQL parser/planner/executor. - -=== Deliverables - -* [x] GQL parser (`core-factor/gql/gql.factor`) - Full PEG-based parser -* [x] Query planner - Cost-based planning with step types -* [x] Executor - In-memory and pluggable persistent storage -* [x] EXPLAIN functionality - EXPLAIN, ANALYZE, VERBOSE modes -* [x] Provenance output - WITH PROVENANCE clause support - -=== Acceptance Criteria - -* ✅ GQL PoC subset fully functional -* ✅ EXPLAIN returns readable plans (including VERBOSE mode) -* ✅ Provenance tracks to journal entries -* ✅ Constraint rejections explained - -== Phase 7: Form.Normalizer (Self-Normalizing) ✅ - -*Status*: Complete (v0.0.4) - -*Goal*: Automatic functional dependency discovery, type-encoded normal forms, and proof-carrying schema evolution. - -=== Deliverables - -* [x] FD discovery algorithm (`normalizer/factor/fd-discovery.factor`) - DFD algorithm -* [x] `INTROSPECT SCHEMA` GQL command - Schema inspection -* [x] Type encoding in Lean 4 (`normalizer/lean/FunDep.lean`) -* [x] Normal form predicates (1NF through BCNF) -* [x] Proposal generation (`denormalization-proposal`) -* [x] Three-phase migration (`normalizer/factor/migration.factor`) -* [x] Query rewriting during migration transitions - -=== Acceptance Criteria - -* ✅ Discover FDs with configurable confidence threshold (three-tier: high/medium/low) -* ✅ FunDep.lean types compatible with GQL-dt -* ✅ Normal form analysis (1NF-BCNF) with violation detection -* ✅ Generate human/agent-readable proposals -* ✅ Three-phase migration (Announce → Shadow → Commit) with rollback -* ✅ Lean4 proofs for verified transformations - -=== Resolved Questions - -* ✅ Q-NORM-001: DFD algorithm chosen (depth-first with partition refinement) -* ✅ Q-NORM-002: Three-tier confidence classification -* ✅ Q-NORM-003: Denormalization proposals with join-based approach -* ✅ Q-NORM-004: CBOR-encoded proof blobs for FFI -* ✅ Q-NORM-005: Query rewriting via migration framework - -See link:spec/self-normalizing.adoc[Self-Normalizing Specification] for full details. - -== Phase 8: Production Hardening ✅ - -*Status*: Complete (v0.0.4) - -*Goal*: Complete PoC with all seam checks passing, production-ready. - -=== Deliverables - -* [x] Seam tests (`core-factor/gql/seam-tests.factor`) - Full pipeline validation -* [x] End-to-end test suite - Parser → Planner → Executor → Normalizer -* [x] Performance benchmarks (`core-factor/gql/benchmarks.factor`) -* [x] Migration tests (`normalizer/factor/migration-tests.factor`) -* [x] Pluggable storage backend (`core-factor/gql/storage-backend.factor`) - -=== Acceptance Criteria - -* ✅ All seam tests pass (full pipeline) -* ✅ Benchmark suite for regression detection -* ✅ Memory/bridge backend abstraction -* ✅ EXPLAIN correlation tests -* ✅ Large dataset stress tests - -== Phase 9: Multi-Protocol API Server ✅ - -*Status*: Complete (v0.0.5) - -*Goal*: Production API server with REST, gRPC, and GraphQL interfaces. - -=== Deliverables - -* [x] API server binary (`lithoglyph-server`) in Zig -* [x] REST API with OpenAPI 3.1 specification (`api/spec/openapi.yaml`) -* [x] gRPC API with Protocol Buffers (`api/proto/lith.proto`) -* [x] GraphQL API with SDL schema (`api/graphql/schema.graphql`) -* [x] WebSocket support for GraphQL subscriptions (`api/src/websocket.zig`) -* [x] Authentication middleware (JWT, API keys) -* [x] Health check and Prometheus metrics endpoints -* [x] Form.Bridge FFI integration (`api/src/bridge_client.zig`) -* [x] Integration tests for all protocols (`api/src/integration_tests.zig`) - -=== REST API Design - -[source] ----- -POST /v1/query # Execute GQL query -POST /v1/collections # Create collection -GET /v1/collections # List collections -GET /v1/collections/{name} # Get collection schema -DELETE /v1/collections/{name} # Drop collection -GET /v1/journal # Query journal entries -POST /v1/normalize/discover # Discover FDs -POST /v1/normalize/analyze # Analyze normal form -POST /v1/migrate/start # Start migration -POST /v1/migrate/commit # Commit migration -GET /v1/health # Health check -GET /v1/metrics # Prometheus metrics ----- - -=== gRPC Service Definition - -[source,protobuf] ----- -service Lith { - rpc Query(QueryRequest) returns (QueryResponse); - rpc CreateCollection(CreateCollectionRequest) returns (CreateCollectionResponse); - rpc ListCollections(Empty) returns (ListCollectionsResponse); - rpc GetJournal(JournalRequest) returns (stream JournalEntry); - rpc DiscoverDependencies(DiscoverRequest) returns (DiscoverResponse); - rpc StartMigration(MigrationRequest) returns (MigrationResponse); -} ----- - -=== GraphQL Schema - -[source,graphql] ----- -type Query { - collections: [Collection!]! - collection(name: String!): Collection - journal(since: Int, limit: Int): [JournalEntry!]! - query(gql: String!, provenance: ProvenanceInput): QueryResult! -} - -type Mutation { - createCollection(name: String!, schema: SchemaInput!): Collection! - execute(gql: String!, provenance: ProvenanceInput!): MutationResult! - startMigration(collection: String!, target: NormalForm!): Migration! -} - -type Subscription { - journalStream(collection: String): JournalEntry! -} ----- - -=== Acceptance Criteria - -* ✅ All GQL operations accessible via REST, gRPC, and GraphQL -* ✅ Provenance tracking across all protocols -* ✅ Streaming support via WebSocket subscriptions (journal streaming) -* ✅ OpenAPI, protobuf, and GraphQL SDL specs generate valid clients -* ✅ Prometheus metrics endpoint -* ✅ Full protobuf serialization/deserialization -* ✅ graphql-ws protocol for subscriptions - -== Phase 10: Language Bindings (M12) - -*Status*: 🚧 Next - -*Target*: v0.0.6 - -*Goal*: Native client libraries for ReScript and PHP. - -*Dependencies*: Phase 9 (HTTP API) complete ✅ - -=== ReScript Client (`@lithoglyph/rescript`) - -* [ ] HTTP client using Fetch API -* [ ] Type-safe GQL query builder -* [ ] Provenance helpers -* [ ] Response type definitions -* [ ] Deno-compatible (no Node.js) - -[source,rescript] ----- -// Example usage -let result = await Lith.query( - ~collection="articles", - ~where=Field("status", Eq, "published"), - ~provenance={actor: "editor@news.org", rationale: "Daily review"}, -) ----- - -=== PHP Client (`lithoglyph/lithoglyph-php`) - -* [ ] PSR-18 HTTP client compatibility -* [ ] Composer package -* [ ] GQL query builder (fluent interface) -* [ ] Provenance handling -* [ ] Laravel/Symfony integration helpers - -[source,php] ----- -query() - ->collection('articles') - ->where('status', '=', 'published') - ->withProvenance('editor@news.org', 'Daily review') - ->execute(); ----- - -=== Acceptance Criteria - -* Both clients pass integration tests against lithoglyph-server -* ReScript client works with Deno -* PHP client installable via Composer -* Documentation with examples - -== Phase 11: CMS Integration (WordPress) - -*Status*: 📋 Planned - -*Goal*: WordPress plugin for Lith-backed content with full provenance. - -*Dependencies*: Phase 10 (PHP client) must be complete. - -=== Deliverables - -* [ ] WordPress plugin (`lithoglyph-wordpress`) -* [ ] Custom post type backed by Lith -* [ ] Edit history with provenance (who changed what, why) -* [ ] Time-travel: view any historical version -* [ ] Revision comparison with narrative diffs -* [ ] Admin UI for Lith connection settings -* [ ] WP-CLI commands for Lith operations - -=== Features - -* **Provenance on Save**: Every post edit records actor + rationale -* **Audit Log**: Complete history visible in admin -* **Rollback**: Restore any previous version with one click -* **Evidence Chain**: For journalism, link sources to claims -* **Multi-site**: Shared Lith across WordPress network - -=== Acceptance Criteria - -* Plugin installable from WordPress admin -* Posts stored in Lith with full provenance -* Revision history shows narrative explanations -* Rollback works without data loss -* Compatible with WordPress 6.x+ - -== Future Phases (Post-PoC) - -=== Form.ControlPlane - -* Gleam/OTP session management -* Cluster coordination -* Production supervision - -=== Distribution - -* Multi-node replication -* Consensus protocol selection -* Network protocol design - -=== Ecosystem - -* Additional language bindings (Rust, Python via HTTP) -* IDE integration (VS Code, JetBrains) -* Visualization tools -* Lith Studio (zero-friction GUI) -* Lith Debugger (proof-carrying debugger) -* Other CMS integrations (Drupal, Ghost, Strapi) - -== Non-Goals (Explicitly Deferred) - -* SQL compatibility layer -* OLAP optimizations -* Full-text search (use integration patterns instead) -* Time-series optimizations -* Real-time subscriptions (use CDC/webhooks instead) - -These may be addressed post-PoC based on community needs. - -== Path to 1.0.0 - -=== Semantic Versioning Plan - -[cols="1,2,3"] -|=== -| Version | Milestone | Focus - -| v0.0.6 | M12 | Language bindings (ReScript, PHP) -| v0.0.7 | M13 | CMS integration (WordPress) -| v0.1.0 | M14 | Form.ControlPlane (clustering, supervision) -| v0.2.0 | - | Distribution (multi-node replication) -| v0.9.0 | M14.5 | Final Naming and Branding (replace "Lith" temporary name) -| v1.0.0-rc | M15 | Release candidate (stability, testing) -| v1.0.0 | GA | General availability -|=== - -=== 1.0.0 Requirements - -* [ ] All M1-M14.5 milestones complete -* [ ] Six months of stability (no breaking changes) -* [ ] Production deployments validated -* [ ] Security audit completed -* [ ] Performance benchmarks published -* [ ] Comprehensive documentation - -== Version History - -[cols="1,1,2"] -|=== -| Version | Date | Milestone - -| v0.0.5 | 2026-01-12 | M11 - Multi-Protocol API Server -| v0.0.4 | 2026-01-12 | M8-M10 - MVP Complete (Form.Runtime + Form.Normalizer) -| v0.0.3 | 2026-01-12 | M7 - Complete Documentation Suite -| v0.0.2 | 2026-01-11 | M1-M6 - Core Specs + Forth PoC -| v0.0.1 | 2026-01-03 | Repository Initialization -|=== - -See link:CHANGELOG.md[CHANGELOG] for detailed release notes. diff --git a/lithoglyph/RSR_OUTLINE.adoc b/lithoglyph/RSR_OUTLINE.adoc deleted file mode 100644 index ede631fe..00000000 --- a/lithoglyph/RSR_OUTLINE.adoc +++ /dev/null @@ -1,218 +0,0 @@ -= RSR Template Repository - -image:[Palimpsest-MPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] image:[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] -:toc: -:sectnums: - -// Badges -image:https://img.shields.io/badge/RSR-Infrastructure-cd7f32[RSR Infrastructure] -image:https://img.shields.io/badge/Phase-Maintenance-brightgreen[Phase] -image:https://img.shields.io/badge/Guix-Primary-purple?logo=gnu[Guix] - -== Overview - -**The canonical template for RSR (Rhodium Standard Repository) projects.** - -This repository provides the standardized structure, configuration, and tooling for all 139 repos in the hyperpolymath ecosystem. Use it to: - -* Bootstrap new projects with RSR compliance -* Reference the standard directory structure -* Copy configuration templates (Justfile, STATE.scm, etc.) - -== Quick Start - -[source,bash] ----- -# Clone the template -git clone https://github.com/hyperpolymath/RSR-template-repo my-project -cd my-project - -# Remove template git history -rm -rf .git -git init - -# Customize -sed -i 's/RSR-template-repo/my-project/g' Justfile guix.scm README.adoc - -# Enter development environment -guix shell -D -f guix.scm - -# Validate compliance -just validate-rsr ----- - -== What's Included - -[cols="1,3"] -|=== -|File/Directory |Purpose - -|`.editorconfig` -|Editor configuration (indent, charset) - -|`.gitignore` -|Standard ignore patterns - -|`.guix-channel` -|Guix channel definition - -|`.well-known/` -|RFC-compliant metadata (security.txt, ai.txt, humans.txt) - -|`docs/` -|Documentation directory - -|`guix.scm` -|Guix package definition - -|`justfile` -|Task runner with 50+ recipes - -|`LICENSE.txt` -|Palimpsest (PMPL-1.0-or-later) - -|`README.adoc` -|This file - -|`RSR_COMPLIANCE.adoc` -|Compliance tracking - -|`STATE.scm` -|Project state checkpoint -|=== - -== Justfile Features - -The template Justfile provides: - -* **~10 billion recipe combinations** via matrix recipes -* **Cookbook generation**: `just cookbook` → `docs/just-cookbook.adoc` -* **Man page generation**: `just man` → `docs/man/project.1` -* **RSR validation**: `just validate-rsr` -* **STATE.scm management**: `just state-touch`, `just state-phase` -* **Container support**: `just container-build`, `just container-push` -* **CI matrix**: `just ci-matrix [stage] [depth]` - -=== Key Recipes - -[source,bash] ----- -just # Show all recipes -just help # Detailed help -just info # Project info -just combinations # Show matrix options - -just build # Build (debug) -just test # Run tests -just quality # Format + lint + test -just ci # Full CI pipeline - -just validate # RSR + STATE validation -just docs # Generate all docs -just cookbook # Generate Justfile docs - -just guix-shell # Guix dev environment -just container-build # Build container ----- - -== Directory Structure - -[source] ----- -project/ -├── .editorconfig # Editor settings -├── .gitignore # Git ignore -├── .guix-channel # Guix channel -├── .well-known/ # RFC metadata -│ ├── ai.txt -│ ├── humans.txt -│ └── security.txt -├── config/ # Nickel configs (optional) -├── docs/ # Documentation -│ ├── generated/ -│ ├── man/ -│ └── just-cookbook.adoc -├── guix.scm # Guix package -├── Justfile # Task runner -├── LICENSE.txt # Dual license -├── README.adoc # Overview -├── RSR_COMPLIANCE.adoc # Compliance -├── src/ # Source code -├── STATE.scm # State checkpoint -└── tests/ # Tests ----- - -== RSR Compliance - -=== Language Tiers - -* **Tier 1** (Gold): Rust, Elixir, Zig, Ada, Haskell, ReScript -* **Tier 2** (Silver): Nickel, Racket, Guile Scheme, Nix -* **Infrastructure**: Guix channels, derivations - -=== Required Files - -* `.editorconfig` -* `.gitignore` -* `justfile` -* `README.adoc` -* `RSR_COMPLIANCE.adoc` -* `LICENSE` (PMPL-1.0-or-later) -* `.well-known/security.txt` -* `.well-known/ai.txt` -* `.well-known/humans.txt` -* `guix.scm` OR `flake.nix` - -=== Prohibited - -* Python outside `salt/` directory -* TypeScript/JavaScript (use ReScript) -* CUE (use Guile/Nickel) -* `Dockerfile` (use `Containerfile`) - -== STATE.scm - -The STATE.scm file tracks project state: - -[source,scheme] ----- -(define state - `((metadata - (project . "my-project") - (updated . "2025-12-10")) - (position - (phase . implementation) ; design|implementation|testing|maintenance|archived - (maturity . beta)) ; experimental|alpha|beta|production|lts - (ecosystem - (part-of . ("RSR Framework")) - (depends-on . ())))) ----- - -== Badge Schema - -Generate badges from STATE.scm: - -[source,bash] ----- -just badges standard ----- - -See `docs/BADGE_SCHEMA.adoc` for the full badge taxonomy. - -== Ecosystem Integration - -This template is part of: - -* **STATE.scm Ecosystem**: Conversation checkpoints -* **RSR Framework**: Repository standards -* **Consent-Aware-HTTP**: .well-known compliance - -== License - -SPDX-License-Identifier: CC-BY-SA-4.0 - -== Links - -* https://github.com/hyperpolymath/elegant-STATE[elegant-STATE] - STATE.scm tooling -* https://github.com/hyperpolymath/conative-gating[conative-gating] - Policy enforcement -* https://rhodium.sh[Rhodium Standard] - RSR documentation diff --git a/lithoglyph/SECURITY.md b/lithoglyph/SECURITY.md deleted file mode 100644 index 21f94f72..00000000 --- a/lithoglyph/SECURITY.md +++ /dev/null @@ -1,234 +0,0 @@ -# Security Policy - - -We take security seriously. We appreciate your efforts to responsibly disclose vulnerabilities and will make every effort to acknowledge your contributions. - -## Table of Contents - -- [Reporting a Vulnerability](#reporting-a-vulnerability) -- [What to Include](#what-to-include) -- [Response Timeline](#response-timeline) -- [Disclosure Policy](#disclosure-policy) -- [Scope](#scope) -- [Safe Harbour](#safe-harbour) -- [Security Updates](#security-updates) -- [Security Architecture](#security-architecture) -- [Development Security](#development-security) - ---- - -## Reporting a Vulnerability - -### Preferred Method: GitHub Security Advisories - -The preferred method for reporting security vulnerabilities is through GitHub's Security Advisory feature: - -1. Navigate to [Report a Vulnerability](https://github.com/hyperpolymath/lithoglyph/security/advisories/new) -2. Click **"Report a vulnerability"** -3. Complete the form with as much detail as possible -4. Submit — we'll receive a private notification - -This method ensures: - -- End-to-end encryption of your report -- Private discussion space for collaboration -- Coordinated disclosure tooling -- Automatic credit when the advisory is published - -### Alternative: Email - -If you cannot use GitHub Security Advisories, you may email us directly: - -| | | -|---|---| -| **Email** | j.d.a.jewell@open.ac.uk | - -> **Important:** Do not report security vulnerabilities through public GitHub issues, pull requests, discussions, or social media. - ---- - -## What to Include - -A good vulnerability report helps us understand and reproduce the issue quickly. - -### Required Information - -- **Description**: Clear explanation of the vulnerability -- **Impact**: What an attacker could achieve (confidentiality, integrity, availability) -- **Affected versions**: Which versions/commits are affected -- **Reproduction steps**: Detailed steps to reproduce the issue - -### Helpful Additional Information - -- **Proof of concept**: Code, scripts, or screenshots demonstrating the vulnerability -- **Attack scenario**: Realistic attack scenario showing exploitability -- **CVSS score**: Your assessment of severity (use [CVSS 3.1 Calculator](https://www.first.org/cvss/calculator/3.1)) -- **CWE ID**: Common Weakness Enumeration identifier if known -- **Suggested fix**: If you have ideas for remediation -- **References**: Links to related vulnerabilities, research, or advisories - -### Example Report Structure - -```markdown -## Summary -[One-sentence description of the vulnerability] - -## Vulnerability Type -[e.g., Buffer overflow, Memory corruption, Input validation, etc.] - -## Affected Component -[File path, function name, API endpoint, etc.] - -## Steps to Reproduce -1. [Step 1] -2. [Step 2] -3. [Step 3] - -## Impact -[What could an attacker achieve?] - -## Suggested Fix -[Optional: Your ideas for remediation] -``` - ---- - -## Response Timeline - -| Stage | Target | -|-------|--------| -| Acknowledgement | Within 48 hours | -| Initial assessment | Within 1 week | -| Fix development | Depends on severity | -| Disclosure | Coordinated, after fix available | - -**Severity-based response:** - -| Severity | Fix Target | -|----------|------------| -| Critical | 72 hours | -| High | 1 week | -| Medium | 2 weeks | -| Low | Next release | - ---- - -## Disclosure Policy - -We follow coordinated disclosure: - -1. Reporter submits vulnerability privately -2. We acknowledge receipt within 48 hours -3. We develop and test a fix -4. We release the fix and publish an advisory -5. Reporter is credited (unless they prefer anonymity) - -We request a 90-day disclosure window. If we cannot fix the issue within 90 days, we will work with the reporter on a reasonable timeline. - ---- - -## Scope - -### In Scope - -- **core-zig bridge** (core-zig/) — C ABI boundary, memory safety -- **core-forth kernel** (core-forth/) — Block storage, journal integrity -- **Idris2 ABI** (src/Lith/) — Type safety, proof soundness -- **Lean 4 normalizer** (normalizer/) — Proof correctness -- **BEAM NIFs** (beam/) — Native function interface safety -- **lith-http** (lith-http/) — HTTP API, authentication, rate limiting -- **GQL-DT** (gql-dt/) — Query injection, type confusion -- **Container configurations** — Image security, secrets management -- **CI/CD workflows** — Supply chain security - -### Out of Scope - -- **studio** (Tauri GUI) — Pre-alpha, returns mock data only -- **api** (Zig HTTP) — Known broken, not deployed -- Third-party dependencies (report upstream) -- Social engineering attacks -- Physical access attacks - ---- - -## Safe Harbour - -We consider security research conducted consistent with this policy to be: - -- Authorized concerning any applicable anti-hacking laws -- Authorized concerning any relevant anti-circumvention laws -- Exempt from restrictions in our Terms of Service that would interfere with conducting security research - -We will not pursue legal action against researchers who: - -- Act in good faith -- Avoid privacy violations, data destruction, or service disruption -- Report findings promptly -- Do not exploit vulnerabilities beyond proof-of-concept - ---- - -## Security Updates - -Security updates are distributed through: - -- GitHub Security Advisories -- CHANGELOG.md entries marked `[SECURITY]` -- Git tags with security notes - ---- - -## Security Architecture - -Lithoglyph is designed with security as a core principle across its multi-language stack: - -### Auditability (Narrative Provenance) -- All mutations are journaled in WAL before commitment to blocks -- Full provenance tracking — every change has who, what, when, why -- Deterministic rendering for human verification -- Cryptographic hashes in provenance chains - -### Reversibility -- Every operation has a defined inverse -- Irreversible operations are explicitly marked and require confirmation -- Complete history is preserved in the journal - -### Formal Verification -- **Idris2 ABI proofs**: Memory layout, alignment, packing verified with dependent types -- **Zero `believe_me`**: Hard invariant — no trust shortcuts in ABI definitions -- **Lean 4 normalization proofs**: Schema correctness verified at compile time -- **GQL-DT**: Query correctness verified with dependent types - -### Memory Safety -- **Zig bridge**: All unsafe casts require `// SAFETY:` comments explaining correctness -- **Rust NIFs**: Safe Rustler 0.35 API, 0 compiler warnings -- **Forth kernel**: Stack-based — bounded memory model - -### Constraint Enforcement -- Constraints enforced at the bridge layer (core-zig) -- Rejections include explanations (not silent failures) -- Parameterized query builders — no SQL injection surface - -### Container Security -- Base images: Chainguard (`cgr.dev/chainguard/wolfi-base:latest`) -- Image signing: cerro-torre with ML-DSA-87 post-quantum crypto -- Secret rotation: rokur with argon2id -- TLS termination: svalinn gateway with policy enforcement -- Runtime verification: vordr formal proof checking - -## Development Security - -- Dependencies are pinned to specific versions/SHAs -- All GitHub Actions workflows use SHA-pinned actions -- SPDX license headers on all source files -- Hypatia neurosymbolic scanner runs on every PR -- Echidnabot detects dangerous patterns (`believe_me`, `sorry`, unsafe casts) -- TruffleHog scans for leaked secrets -- OpenSSF Scorecard measures supply chain health - -## Supported Versions - -| Version | Supported | -|---------|-----------| -| 0.0.7 (current) | Development — security fixes applied | -| < 0.0.7 | Not supported | diff --git a/lithoglyph/TOPOLOGY.md b/lithoglyph/TOPOLOGY.md deleted file mode 100644 index e233ec40..00000000 --- a/lithoglyph/TOPOLOGY.md +++ /dev/null @@ -1,126 +0,0 @@ - - - -# TOPOLOGY.md — Lithoglyph - -## System Architecture - -``` - ┌─────────────────────────────────────┐ - │ svalinn (TLS gateway) │ - │ ML-DSA-87 · policy: strict │ - └────────────────┬────────────────────┘ - │ :8443 - ┌────────────────────────┼────────────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ - │ lith-http │ │ api (Zig) │ │ studio (Tauri) │ - │ Elixir/Phoenix │ │ REST + gRPC │ │ Desktop GUI │ - │ :4000 │ │ :8080 (BROKEN) │ │ │ - └────────┬─────────┘ └────────┬─────────┘ └────────┬─────────┘ - │ │ │ - └──────────────────────┼───────────────────────┘ - │ - ┌─────────────────────────────▼─────────────────────────────┐ - │ Elixir/OTP Control Plane │ - │ ┌──────────────┐ ┌──────────────┐ ┌────────────────┐ │ - │ │ Supervision │ │ Clustering │ │ BEAM NIFs │ │ - │ │ Trees │ │ (planned) │ │ Zig + Rust │ │ - │ └──────────────┘ └──────────────┘ └───────┬────────┘ │ - └──────────────────────────────────────────────┼────────────┘ - │ C ABI - ┌──────────────────────────────────────────────▼────────────┐ - │ core-zig (Bridge) │ - │ 19 functions · WAL commit · block allocator · compaction │ - │ ┌──────────┐ ┌──────────┐ ┌──────────────────────┐ │ - │ │ bridge │ │ blocks │ │ schema introspection │ │ - │ │ .zig │ │ .zig │ │ + proof verifier │ │ - │ └──────────┘ └──────────┘ └──────────────────────┘ │ - └──────────────────────────────┬────────────────────────────┘ - │ - ┌──────────────────────────────▼────────────────────────────┐ - │ core-forth (Kernel) │ - │ Block storage · Journal · Data model │ - │ 17/17 tests pass │ - └──────────────────────────────────────────────────────────-─┘ - - ┌─────────────────────────────────────────────────────────┐ - │ Verification Layers │ - │ │ - │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ - │ │ Idris2 ABI │ │ Lean 4 │ │ GQL-DT │ │ - │ │ src/Lith/ │ │ normalizer/ │ │ gql-dt/ │ │ - │ │ 3 files │ │ 52 proofs │ │ Lean 4 │ │ - │ │ 0 believe_me│ │ FD discovery│ │ type-safe │ │ - │ └──────────────┘ └──────────────┘ │ queries │ │ - │ └──────────────┘ │ - │ ┌──────────────┐ ┌──────────────┐ │ - │ │ Factor │ │ Glyphbase │ │ - │ │ core-factor/│ │ glyphbase/ │ │ - │ │ GQL runtime │ │ graph store │ │ - │ └──────────────┘ └──────────────┘ │ - └─────────────────────────────────────────────────────────┘ - - Data flow: - mutation → core-forth blocks → core-zig bridge (lith_*) → BEAM NIF → lith-http API - query → GQL-DT (Lean verify) → Factor GQL → core-zig → core-forth → result - glyphbase NIF → core-zig bridge (19 functions, LgBlob/LgStatus types) → core-forth - - Naming: fdb_* → lith_*, FQL/FBQL/FDQL → GQL (Glyph Query Language), FormBD → Lithoglyph -``` - -## Completion Dashboard - -| Component | Progress | Status | -|------------------------|------------------------------|----------------| -| core-forth (kernel) | `██████████` 100% | Complete | -| core-zig (bridge) | `██████████` 100% | Complete | -| ffi/zig (delegation) | `██████████` 100% | Complete | -| Idris2 ABI (proofs) | `██████████` 100% | Complete | -| Lean 4 normalizer | `██████████` 100% | Complete | -| core-factor (GQL) | `██████████` 100% | Complete | -| BEAM NIF (Zig) | `████████░░` 80% | Builds | -| BEAM NIF (Rust) | `████████░░` 80% | Builds | -| glyphbase NIF | `██████░░░░` 60% | Linked to core | -| lith-http (Elixir) | `█████████░` 90% | M15 complete | -| gql-dt (Lean 4) | `████████░░` 80% | Needs audit | -| glyphbase | `████████░░` 80% | Needs audit | -| api (Zig HTTP) | `██████████` 100% | L1 complete | -| studio (Tauri) | `██░░░░░░░░` 20% | Mock data | -| Containerfile | `██████████` 100% | Complete | -| selur-compose | `██████████` 100% | Complete | -| IP rename | `████████░░` 80% | fdb→lith done | -| **Overall** | `████████░░` **80%** | | - -## Key Dependencies - -``` -lithoglyph -├── gforth (Forth kernel — block storage) -├── zig 0.15.2 (C ABI bridge, BEAM NIF) -├── idris2 (dependent-type ABI proofs) -├── lean 4 v4.15.0 (normalization, GQL-DT) -├── mathlib v4.15.0 (GQL-DT proofs) -├── factor (GQL runtime — parser, planner, executor) -├── rustler 0.35 (Rust BEAM NIF) -├── elixir 1.18 / OTP 27 (lith-http, control plane) -├── phoenix (HTTP API framework) -├── tauri 2.0+ (studio desktop GUI) -├── nickel (configuration) -│ -├── Container ecosystem: -│ ├── svalinn (TLS gateway + policy enforcement) -│ ├── vordr (runtime verification) -│ ├── cerro-torre (image signing, ML-DSA-87) -│ ├── rokur (secret rotation, argon2id) -│ └── selur-compose (deployment orchestration) -│ -├── Sibling databases: -│ ├── verisimdb (octad database — shared GQL patterns) -│ ├── quandledb (knot-theoretic database) -│ └── nqc (normal-form query compiler) -│ -└── Safety: - └── proven (formally verified SafeString, SafeJson, etc.) -``` diff --git a/lithoglyph/VERSIONING.adoc b/lithoglyph/VERSIONING.adoc deleted file mode 100644 index ff13dfe5..00000000 --- a/lithoglyph/VERSIONING.adoc +++ /dev/null @@ -1,660 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Versioning & Stability Policy -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font -:sectnums: - -[.lead] -API stability guarantees, versioning conventions, and compatibility promises for Lith. - -toc::[] - -== Overview - -Lith takes stability seriously. As a database designed for long-term archives and audit trails, we understand that your data and queries must remain accessible for years or decades. This document describes our versioning scheme, stability guarantees, and migration policies. - -[IMPORTANT] -==== -**Current Status: Pre-1.0 Development** - -Lith is currently at version **0.0.1** (conceptual + PoC stage). All APIs, formats, and interfaces are subject to change. Once we reach 1.0, the stability guarantees in this document become binding commitments. -==== - -== Version Numbering - -Lith follows https://semver.org/[Semantic Versioning 2.0.0]: - ----- -MAJOR.MINOR.PATCH[-PRERELEASE][+BUILD] - -Examples: - 0.1.0 Pre-1.0 development - 1.0.0 First stable release - 1.2.3 Stable release with patches - 2.0.0-alpha.1 Major version pre-release - 1.5.0+build.42 Build metadata (informational) ----- - -=== Version Components - -[cols="1,3"] -|=== -| Component | Meaning - -| **MAJOR** -| Incremented for incompatible API changes. Requires migration effort. - -| **MINOR** -| Incremented for backwards-compatible new features. Safe to upgrade. - -| **PATCH** -| Incremented for backwards-compatible bug fixes. Safe to upgrade. - -| **PRERELEASE** -| Alpha, beta, or release candidate (e.g., `-alpha.1`, `-beta.2`, `-rc.1`) - -| **BUILD** -| Build metadata, ignored for version precedence (e.g., `+git.abc123`) -|=== - -=== Pre-1.0 Versioning (Current) - -During pre-1.0 development: - -* **0.MINOR.PATCH** format -* MINOR increments may include breaking changes -* PATCH increments are bug fixes only -* No stability guarantees, but we document all breaking changes -* Migration scripts provided when feasible - -=== Post-1.0 Versioning (Future) - -After 1.0 release: - -* Breaking changes **only** in MAJOR versions -* MINOR versions are always backwards-compatible -* PATCH versions contain only bug fixes and security patches -* Deprecation warnings precede removal by at least one MINOR version - -== Stability Tiers - -Each Lith component is assigned a stability tier that defines its compatibility guarantees. - -=== Tier 1: Stable - -[cols="1,4"] -|=== -| Guarantee | No breaking changes within a major version - -| Deprecation -| 6-month warning period before removal - -| Migration -| Automated migration tools provided - -| Documentation -| Complete API documentation maintained -|=== - -**Post-1.0 Stable Components:** - -* GQL core query syntax -* Core data types (STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, JSON, PROMPT_SCORE) -* Provenance model (actor, rationale, timestamp) -* Document and edge CRUD operations -* Journal read operations -* INTROSPECT and EXPLAIN commands -* Error response format - -=== Tier 2: Provisional - -[cols="1,4"] -|=== -| Guarantee | API may change in minor versions with deprecation warnings - -| Deprecation -| 3-month warning period - -| Migration -| Migration guidance documented - -| Documentation -| API documented but marked as provisional -|=== - -**Provisional Components (stabilizing):** - -* GQL advanced syntax (TRAVERSE depth options, complex WHERE clauses) -* Constraint definition syntax -* Schema evolution operations -* HTTP API endpoints -* Zig ABI function signatures - -=== Tier 3: Experimental - -[cols="1,4"] -|=== -| Guarantee | May change or be removed in any release - -| Deprecation -| No deprecation period required - -| Migration -| Best-effort migration guidance - -| Documentation -| Documented with experimental warnings -|=== - -**Experimental Components:** - -* FQLdt (dependently-typed queries) -* Self-normalizing features (Form.Normalizer) -* Distributed consensus (multi-node) -* Proof-carrying schema evolution -* Graph analytics extensions - -=== Tier 4: Internal - -[cols="1,4"] -|=== -| Guarantee | Implementation detail, not part of public API - -| Deprecation -| None - -| Migration -| Not applicable - -| Documentation -| Internal documentation only -|=== - -**Internal Components:** - -* Block header layout (beyond documented format) -* Journal segment encoding details -* CRC32C table implementation -* Memory buffer management -* Forth word definitions (internal) - -== Component Stability Matrix - -=== Current Status (v0.x) - -[cols="2,1,2,2"] -|=== -| Component | Tier | Status | Notes - -| GQL Core Syntax -| Provisional -| Stabilizing -| Grammar mostly stable, minor changes expected - -| GQL Advanced Syntax -| Experimental -| Active development -| TRAVERSE, complex predicates evolving - -| Block Format -| Internal -| Stable internally -| 4 KiB blocks, 64-byte headers locked - -| Journal Format -| Internal -| Stable internally -| Append-only, sequence-numbered - -| Provenance Model -| Stable -| Locked -| actor + rationale required, extensions allowed - -| PROMPT_SCORE Type -| Stable -| Locked -| 0-100 integer, semantics defined - -| Zig ABI -| Provisional -| Stabilizing -| Function signatures may change - -| Error Format -| Provisional -| Stabilizing -| JSON structure mostly stable - -| HTTP API -| Experimental -| Planned -| Not yet implemented - -| FQLdt -| Experimental -| Research -| Lean 4 integration in progress - -| Self-Normalizing -| Experimental -| Research -| FD discovery algorithms being evaluated -|=== - -=== Planned 1.0 Status - -[cols="2,1,2"] -|=== -| Component | Target Tier | Criteria for Stability - -| GQL Core Syntax -| Stable -| Grammar frozen, 100% backwards compatible - -| GQL Advanced Syntax -| Provisional -| Core features stable, extensions provisional - -| Block Format -| Internal -| Versioned, migration tools available - -| Journal Format -| Internal -| Versioned, replay guaranteed - -| Provenance Model -| Stable -| Schema frozen - -| Zig ABI -| Stable -| ABI frozen, soname versioned - -| Error Format -| Stable -| JSON schema frozen - -| HTTP API -| Provisional -| OpenAPI spec published -|=== - -== Format Versioning - -Lith uses internal version numbers for on-disk formats to ensure data remains readable across upgrades. - -=== Block Format Version - -[source,text] ----- -Block Header Offset 4-5: version (2 bytes) - -Current: 0x0001 (v1) ----- - -**Version Compatibility:** - -[cols="1,1,3"] -|=== -| Read Version | Write Version | Behavior - -| v1 | v1 | Full read/write -| v2 | v1 | Read-only (upgrade required for writes) -| v1 | v2 | Automatic upgrade on first write (with backup) -|=== - -=== Journal Format Version - -[source,text] ----- -Journal Segment Header: schema_version field - -Current: 1 ----- - -**Journal Replay Guarantee:** - -Journals written by any Lith version can be replayed by any later version. This is a **permanent guarantee**—we will never break journal replay. - -=== Schema Version - -Each collection tracks its schema version: - -[source,gql] ----- -INTROSPECT SCHEMA evidence; - -{ - "collection": "evidence", - "schema_version": 3, - "created_at": "2024-01-15T...", - "migrations": [ - {"from": 1, "to": 2, "applied": "2024-03-01T..."}, - {"from": 2, "to": 3, "applied": "2024-06-15T..."} - ] -} ----- - -== API Versioning - -=== GQL Versioning - -GQL syntax is versioned at the dialect level: - -[source,gql] ----- --- Explicitly request GQL v1 syntax -#gql-version 1 - -SELECT * FROM evidence; ----- - -Without a version directive, the latest stable version is assumed. - -=== Zig ABI Versioning - -The Zig ABI uses soname versioning: - -[source,text] ----- -liblith.so.1 # ABI version 1 -liblith.so.1.2.3 # Full version ----- - -ABI changes: - -* **PATCH**: Bug fixes, no signature changes -* **MINOR**: New functions added, existing signatures unchanged -* **MAJOR**: Breaking signature changes - -=== HTTP API Versioning - -The HTTP API uses URL path versioning: - -[source,text] ----- -POST /v1/query -GET /v1/collections -GET /v2/collections (future, with breaking changes) ----- - -Multiple API versions may be supported simultaneously. - -== Deprecation Policy - -=== Deprecation Process - -1. **Announcement**: Feature marked deprecated in CHANGELOG and documentation -2. **Warning Period**: Runtime warnings emitted when deprecated feature is used -3. **Migration Guide**: Documentation explains how to migrate -4. **Removal**: Feature removed in next major version (post-1.0) - -=== Deprecation Timelines - -[cols="1,2,2"] -|=== -| Tier | Warning Period | Removal - -| Stable -| 6 months minimum -| Next major version - -| Provisional -| 3 months minimum -| Next minor version (with major bump) - -| Experimental -| 1 release minimum -| Any release - -| Internal -| None required -| Any release -|=== - -=== Deprecation Warnings - -Deprecated features emit warnings: - -[source,json] ----- -{ - "status": "success", - "warnings": [ - { - "type": "deprecation", - "feature": "TRAVERSE without DEPTH", - "message": "Implicit DEPTH 1 is deprecated. Specify DEPTH explicitly.", - "deprecated_in": "1.2.0", - "removal_in": "2.0.0", - "migration": "Add 'DEPTH 1' to your TRAVERSE clauses" - } - ], - "result": {...} -} ----- - -== Migration Policy - -=== Data Migration - -Lith provides migration tools for format changes: - -[source,bash] ----- -# Check if migration is needed -lith doctor --check-version mydb/ - -# Backup before migration -lith backup mydb/ mydb-backup-$(date +%Y%m%d)/ - -# Run migration -lith migrate mydb/ --to-version 2 - -# Verify migration -lith doctor --verify mydb/ ----- - -=== Query Migration - -For GQL syntax changes: - -[source,bash] ----- -# Check queries for deprecated syntax -lith lint --queries queries/*.gql - -# Auto-fix where possible -lith lint --fix --queries queries/*.gql - -# Generate migration report -lith lint --report queries/*.gql > migration-report.md ----- - -=== Zero-Downtime Migration - -For production systems: - -1. **Shadow Mode**: New version runs alongside old, receiving replicated writes -2. **Validation**: Queries verified to produce identical results -3. **Cutover**: Traffic switched to new version -4. **Rollback**: Automatic if errors detected - -== Release Cadence - -=== Pre-1.0 (Current) - -* **Milestone releases**: When significant features complete -* **No fixed schedule**: Quality over cadence -* **Breaking changes**: Documented in each release - -=== Post-1.0 (Planned) - -[cols="1,2,2"] -|=== -| Release Type | Cadence | Contents - -| **PATCH** -| As needed (security: within 48 hours) -| Bug fixes, security patches - -| **MINOR** -| Quarterly -| New features, deprecations announced - -| **MAJOR** -| Annual (maximum) -| Breaking changes, deprecations removed -|=== - -=== Long-Term Support (LTS) - -Post-1.0, selected releases receive extended support: - -[cols="1,2,2"] -|=== -| Release Type | Support Period | Updates Include - -| **Regular** -| Until next minor release -| Security + bug fixes - -| **LTS** -| 3 years from release -| Security + critical bug fixes - -| **Extended LTS** -| 5 years (enterprise) -| Security fixes only -|=== - -== Compatibility Guarantees - -=== Forward Compatibility - -Data written by older versions can be read by newer versions: - -[cols="1,1,1"] -|=== -| Written By | Read By | Guaranteed - -| v1.x | v1.y (y > x) | Yes -| v1.x | v2.x | Yes -| v1.x | v3.x | Yes (indefinitely) -|=== - -=== Backward Compatibility - -Queries written for older versions continue to work: - -[cols="1,1,1"] -|=== -| Query Version | Runtime Version | Guaranteed - -| GQL v1 | Lith 1.x | Yes -| GQL v1 | Lith 2.x | Yes (with version directive) -| GQL v1 | Lith 3.x | Yes (with version directive) -|=== - -=== Journal Replay Guarantee - -**Permanent Commitment**: Any journal written by any Lith version can be replayed by any later version. This enables: - -* Point-in-time recovery across upgrades -* Audit trail preservation -* Long-term archive integrity - -== Version Support Matrix - -=== Current Versions - -[cols="1,1,2,2"] -|=== -| Version | Status | Released | End of Support - -| 0.0.1 -| Development -| 2024-01 -| N/A (pre-release) -|=== - -=== Planned Versions - -[cols="1,1,2"] -|=== -| Version | Target | Milestone - -| 0.1.0 -| PoC Complete -| Single-node CRUD, journal, basic GQL - -| 0.5.0 -| Alpha -| Full GQL, constraints, edges - -| 1.0.0 -| Stable -| Production-ready, stability guarantees active -|=== - -== Checking Versions - -=== Runtime Version - -[source,gql] ----- -INTROSPECT VERSION; - -{ - "lith": "0.0.1", - "gql": "1", - "block_format": 1, - "journal_format": 1, - "zig_abi": "0.1.0" -} ----- - -=== Database Version - -[source,gql] ----- -INTROSPECT DATABASE; - -{ - "path": "/data/mydb", - "created_by": "0.0.1", - "last_opened_by": "0.0.1", - "block_format_version": 1, - "journal_entries": 1523, - "collections": 5 -} ----- - -=== Compatibility Check - -[source,bash] ----- -lith doctor --compatibility mydb/ - -Lith Compatibility Report -=========================== -Database created by: 0.0.1 -Current Lith version: 0.1.0 - -Block format: v1 (compatible) -Journal format: v1 (compatible) -Schema versions: all compatible - -Status: COMPATIBLE (no migration required) ----- - -== See Also - -* link:CHANGELOG.md[Changelog] - Release history and breaking changes -* link:ROADMAP.adoc[Roadmap] - Planned features and versions -* link:docs/DEPLOYMENT.adoc[Deployment Guide] - Production deployment -* link:ARCHITECTURE.adoc[Architecture] - Component details diff --git a/lithoglyph/WHITEPAPER.md b/lithoglyph/WHITEPAPER.md deleted file mode 100644 index 44e997d5..00000000 --- a/lithoglyph/WHITEPAPER.md +++ /dev/null @@ -1,440 +0,0 @@ -# SPDX-License-Identifier: CC-BY-SA-4.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) - -# GQL: A Narrative-First Query Language for Provenance-Aware Data - -**Author:** Jonathan D.A. Jewell -**Version:** 1.0 -**Date:** 2026-03-14 -**Status:** Alpha (65% complete) - ---- - -## Abstract - -We present GQL (Glyph Query Language), the query language for Lithoglyph, a -multi-model database designed for domains where understanding *why* data exists -matters as much as the data itself. Traditional query languages treat data as -inert records: rows to be selected, filtered, and aggregated. GQL treats data -as *narrative artefacts*—entities with provenance (who added them), rationale -(why they were added), reversibility (how to undo them), and epistemological -metadata (how much they should be trusted). This paper presents GQL's narrative- -first design philosophy, its two-tier architecture (GQL for users, GQL-DT with -dependent types for developers), its mandatory provenance tracking, its -reversibility-by-default semantics, and its application to investigative -journalism, governance, and interactive documentary. - ---- - -## 1. Introduction - -### 1.1 The Narrative Gap in Database Systems - -Every piece of data in a database has a story: who created it, why it was -created, what evidence supports it, and what would happen if it were removed. -Traditional databases discard this story at insertion time, storing only the -data itself. The narrative—the context that makes data meaningful—is relegated -to application code, external logs, or institutional memory. - -This works when databases serve as accounting ledgers: the numbers speak for -themselves. But in domains where data is *contested*, *evolving*, or -*consequential*—investigative journalism, public policy, scientific research, -legal proceedings—the narrative is not ancillary. It is the primary value. - -Consider an investigative journalist building a database of financial -transactions to support a story about fraud. For each entry, they need to -record: - -- **Who added it?** (The journalist, a source, a scraping tool?) -- **Why?** (Primary evidence, corroboration, background context?) -- **How trustworthy is it?** (Official government record vs. anonymous tip?) -- **Can it be retracted?** (If the source recants, what happens?) -- **Who else has seen it?** (Audit trail for legal discovery.) - -In SQL, all of this must be encoded as application-level conventions: extra -columns, audit tables, trigger-based logging. None of it is *required* by the -language. In GQL, all of it is *mandatory by grammar*. - -### 1.2 Design Thesis - -GQL's design thesis is: - -> **The database is part of the story, not an opaque substrate.** - -This thesis has six concrete implications, which form GQL's design pillars: - -1. **Provenance by construction:** You cannot insert data without specifying - who added it and why. -2. **Reversibility as first-class:** Every operation has a defined inverse, - or is explicitly marked irreversible with justification. -3. **Constraints as ethics:** When constraints fail, GQL explains *why* the - constraint exists, not just that it failed. -4. **Explain everything:** `EXPLAIN` returns not just query plans but - constraint reasoning and provenance chains. -5. **Results carry provenance:** Query results optionally include metadata - about who added each piece of data and when. -6. **Schema is narrative:** Schema changes are recorded events with rationale, - not silent infrastructure updates. - -### 1.3 Contributions - -1. **Mandatory provenance** in query language grammar, not application - conventions (Section 3). -2. **Reversibility-by-default** with proof-carrying inverses (Section 4). -3. **Two-tier design:** GQL (accessible) and GQL-DT (dependently typed) for - different user populations (Section 5). -4. **PROMPT epistemological framework** for source trustworthiness assessment - (Section 6). -5. **Interactive documentary** as a first-class database application pattern - (Section 7). -6. **Formal verification** of schema evolution using Lean 4 and Idris2 - (Section 8). - ---- - -## 2. Architecture - -### 2.1 Lithoglyph's Multi-Model Design - -Lithoglyph is a multi-model database supporting three data paradigms: - -| Paradigm | GQL Syntax | Use Case | -|----------|-----------|----------| -| **Document** | `INSERT INTO collection { ... }` | Unstructured evidence, reports | -| **Edge** | `INSERT EDGE INTO relationship { ... }` | Connections between entities | -| **Relational** | `CREATE COLLECTION (...) WITH DEPENDENT_TYPES` | Structured, typed data | - -GQL operates natively across all three paradigms. Unlike SQL (relational only) -or GraphQL (API layer), GQL is a *database-native* language that treats graphs, -documents, and relations as equal citizens. - -### 2.2 Implementation Stack - -| Layer | Language | Purpose | -|-------|----------|---------| -| Storage (Form.Blocks) | Forth | Fixed-size blocks, append-only journal | -| Data Model (Form.Model) | Forth | Collections, edges, schema, constraints | -| Bridge (Form.Bridge) | Zig | C-ABI bridge, WAL commit, block allocator | -| Runtime (Form.Runtime) | Factor | GQL parser, planner, executor | -| Normalizer | Factor + Lean 4 | FD discovery, normal forms, proof-carrying evolution | -| GQL-DT | Lean 4 | Dependent type checking for GQL-DT tier | -| Control Plane | Elixir/OTP | Sessions, supervision, clustering | - -The choice of Forth for storage and Factor for the runtime reflects Lithoglyph's -design metaphor: "Forth code sculpts data onto disk, carving each operation -into permanent, auditable stone." Concatenative languages make the operation -sequence explicit—every action is a word on the stack, visible and auditable. - ---- - -## 3. Mandatory Provenance - -### 3.1 Provenance in Grammar - -In SQL, provenance is optional. In GQL, it is syntactically mandatory: - -```gql --- SQL (provenance is absent) -INSERT INTO evidence (title) VALUES ('ONS Report'); - --- GQL (provenance is required by grammar) -INSERT INTO evidence (title) VALUES ('ONS Report') -ADDED_BY "researcher_alice" -RATIONALE "Primary source for inflation analysis"; -``` - -Omitting `ADDED_BY` or `RATIONALE` is a *syntax error*, not a best-practice -violation. This ensures that provenance is never accidentally omitted, even by -novice users or automated tools. - -### 3.2 Provenance in Results - -Query results can include provenance metadata: - -```gql -SELECT title, source_url FROM evidence WITH PROVENANCE; - --- Returns: --- { title: "ONS Report", source_url: "...", --- _provenance: { actor: "researcher_alice", --- added_at: "2026-03-14T10:00:00Z", --- rationale: "Primary source for inflation analysis", --- journal_entry: 42 } } -``` - -### 3.3 Provenance Integrity - -Provenance is stored in an append-only journal alongside the data. The journal -is the authoritative audit trail—it cannot be modified, only appended to. -Each journal entry records: - -- **Actor:** Who performed the operation. -- **Timestamp:** When it was performed. -- **Operation:** What was done (INSERT, UPDATE, DELETE, SCHEMA_CHANGE). -- **Rationale:** Why it was done. -- **Inverse:** How to undo it (see Section 4). - ---- - -## 4. Reversibility by Default - -### 4.1 Every Operation Has an Inverse - -GQL requires that every mutation operation has a defined inverse: - -| Operation | Automatic Inverse | -|-----------|------------------| -| `INSERT` | `DELETE` with matching criteria | -| `UPDATE` | `UPDATE` with previous values | -| `DELETE` | `INSERT` with deleted data | -| Schema `ADD COLUMN` | Schema `DROP COLUMN` | -| Schema `DROP COLUMN` | Schema `ADD COLUMN` with data restoration | - -Inverses are computed automatically and stored in the journal. At any point, -a user can roll back operations: - -```gql -ROLLBACK JOURNAL ENTRY 42 -RATIONALE "Source retracted claim"; -``` - -### 4.2 Explicit Irreversibility - -When an operation genuinely cannot be reversed (e.g., GDPR deletion, classified -material purge), GQL requires explicit acknowledgment: - -```gql -DELETE FROM evidence WHERE id = "secret-doc" -IRREVERSIBLE BECAUSE "GDPR Article 17 right to erasure" -AUTHORISED_BY "data_protection_officer"; -``` - -This creates a *tombstone* in the journal recording that data was permanently -deleted, by whom, and why—even though the data itself is gone. - ---- - -## 5. Two-Tier Design - -### 5.1 Tier 1: GQL (Accessible) - -GQL's user-facing syntax is SQL-like with mandatory provenance: - -```gql -INSERT INTO evidence (title, confidence) -VALUES ('ONS Data', 95) -ADDED_BY "researcher_alice" -RATIONALE "Official statistics"; - -SELECT claim, source, prompt_score -FROM evidence -WHERE prompt_score >= 80; -``` - -This tier performs runtime type checking and constraint validation. Errors -include explanations: - -``` -CONSTRAINT_VIOLATION: - constraint: evidence.source_url UNIQUE - reason: "Document with source_url already exists" - constraint_rationale: "Each source entered once to prevent duplicate counting" - suggestion: "Use UPDATE to modify existing document" -``` - -### 5.2 Tier 2: GQL-DT (Dependently Typed) - -GQL-DT extends GQL with Lean 4-style dependent types for compile-time -verification: - -```lean -INSERT INTO evidence ( - title : NonEmptyString, - prompt_provenance : BoundedNat 0 100 -) -VALUES ( - NonEmptyString.mk "ONS Data" (by decide), - BoundedNat.mk 0 100 95 (by omega) (by omega) -) -RATIONALE "Official statistics" -WITH_PROOF { - scores_in_bounds: by lithoglyph_prompt, - provenance_tracked: by lithoglyph_prov -}; -``` - -In GQL-DT: - -- **Types carry proofs:** `BoundedNat 0 100` is a natural number with a - *compile-time proof* that it is between 0 and 100. -- **NonEmptyString** is a string with a proof of non-emptiness. -- **WITH_PROOF** attaches formal theorems to operations. -- **Schema changes carry normalization proofs**: Adding a column can require - proof that it preserves normal form. - -### 5.3 When to Use Each Tier - -| User | Tier | Reason | -|------|------|--------| -| Journalist | GQL | Intuitive SQL-like syntax, runtime safety | -| Researcher | GQL | Familiar syntax, provenance tracking | -| Database admin | GQL-DT | Schema evolution with formal guarantees | -| Application developer | GQL-DT | Compile-time query validation | -| AI agent | GQL-DT | Machine-verifiable proofs for trust | - ---- - -## 6. PROMPT Epistemological Framework - -### 6.1 Source Assessment - -Lithoglyph integrates the PROMPT framework (Provenance, Relevance, -Objectivity, Method, Publication, Timeliness + Transparency) for source -quality assessment: - -```gql -INSERT INTO evidence ( - title, source_url, - prompt_provenance, prompt_relevance, prompt_objectivity, - prompt_method, prompt_publication, prompt_timeliness -) -VALUES ( - 'ONS CPI Data', 'https://ons.gov.uk/...', - 95, 90, 85, 92, 98, 88 -) -ADDED_BY "researcher_alice" -RATIONALE "Primary statistical source"; -``` - -PROMPT scores are first-class data, not application-level metadata. They can -be queried, aggregated, and used in constraint definitions: - -```gql --- Only trust highly-scored sources -SELECT * FROM evidence -WHERE prompt_overall >= 80 -ORDER BY prompt_provenance DESC; - --- Constraint: minimum quality for publication -CREATE CONSTRAINT evidence_quality - ON evidence - CHECK (prompt_overall >= 60) - RATIONALE "Published claims require minimum source quality"; -``` - ---- - -## 7. Interactive Documentary - -### 7.1 Motivation - -Interactive documentary (i-doc) is a form of non-fiction storytelling that -allows audiences to navigate evidence and form their own conclusions, rather -than following a fixed linear narrative. Lithoglyph is purpose-built for i-doc -because its provenance-first design maps directly to the requirements of -transparent, navigable evidence presentation. - -### 7.2 Multi-Perspective Navigation - -GQL supports *navigation paths*—predefined routes through the evidence graph -tailored to different audiences: - -```gql -CREATE NAVIGATION_PATH skeptic_path - START FROM evidence WHERE prompt_method >= 90 - TRAVERSE supports OUTBOUND - ORDER BY prompt_provenance DESC - RATIONALE "For skeptical audiences: start with methodology"; - -CREATE NAVIGATION_PATH policymaker_path - START FROM evidence WHERE prompt_publication >= 95 - TRAVERSE supports OUTBOUND - ORDER BY prompt_timeliness DESC - RATIONALE "For policymakers: start with authoritative, recent sources"; -``` - -### 7.3 Boundary Objects - -Following Star & Griesemer (1989), evidence items can serve as *boundary -objects*—artefacts that are shared across communities but interpreted -differently. GQL's provenance tracking naturally supports this: the same -evidence item carries multiple rationale entries from different actors, -preserving each community's interpretation without forcing consensus. - ---- - -## 8. Formal Verification - -### 8.1 Schema Normalisation - -Lithoglyph's normaliser (implemented in Factor + Lean 4) provides formally -verified schema evolution: - -- **Functional dependency discovery:** Automatically identifies FDs in data. -- **Normal form predicates:** Proves that schema satisfies 1NF, 2NF, 3NF, - or BCNF. -- **Proof-carrying evolution:** Schema changes carry proofs that normalization - is preserved (or intentionally violated with justification). - -52 Lean 4 proofs verify normaliser correctness. - -### 8.2 Idris2 ABI - -Lithoglyph's ABI is formally specified in Idris2, following the hyperpolymath -ABI/FFI standard: - -- 28 Idris2 proof files verify type safety of the ABI. -- Zig FFI bridge (19 functions) connects Forth storage to Factor runtime. -- Zero `believe_me` or `assert_total` in proof code. - ---- - -## 9. Comparison with Existing Systems - -| Feature | SQL | GraphQL | Cypher | GQL | -|---------|-----|---------|--------|-----| -| Provenance | Optional | No | No | **Mandatory** | -| Reversibility | Transactions | No | No | **Every operation** | -| Constraint explanation | Error codes | Error messages | Error messages | **Rationale + suggestion** | -| Source quality | Application-level | No | No | **PROMPT framework** | -| Dependent types | No | No | No | **GQL-DT tier** | -| Multi-model | Relational only | API layer | Graph only | **Document + Edge + Relational** | -| Schema evolution proofs | No | No | No | **Lean 4 verified** | -| Narrative navigation | No | No | No | **Navigation paths** | - ---- - -## 10. Conclusion - -GQL demonstrates that query languages can and should encode domain-specific -integrity requirements in their grammar, not delegate them to application -conventions. For domains where data provenance, source trustworthiness, and -auditability are primary concerns—journalism, governance, research, legal— -GQL's mandatory provenance and reversibility-by-default provide guarantees -that SQL-based systems cannot match without extensive application-level -engineering. - -The two-tier design (GQL + GQL-DT) makes these guarantees accessible to both -non-technical users (who benefit from runtime checking and explanatory errors) -and developers (who benefit from compile-time verification and proof -attachment). The PROMPT framework and interactive documentary support -demonstrate that database systems can be active participants in knowledge -construction, not passive storage. - ---- - -## References - -1. Buneman, P. et al. (2001). "Why and Where: A Characterization of Data - Provenance." *ICDT 2001*, 316–330. -2. Cheney, J. et al. (2009). "Provenance in Databases: Why, How, and Where." - *Foundations and Trends in Databases*, 1(4), 379–474. -3. de Moura, L. & Ullrich, S. (2021). "The Lean 4 Theorem Prover and - Programming Language." *CADE 2021*, 625–635. -4. Gaudenzi, S. (2013). "The Living Documentary: From Representing Reality to - Co-creating Reality in Digital Interactive Documentary." PhD Thesis, - Goldsmiths, University of London. -5. Star, S. L. & Griesemer, J. R. (1989). "Institutional Ecology, 'Translations' - and Boundary Objects." *Social Studies of Science*, 19(3), 387–420. -6. ISO/IEC 39075:2024. "Information technology — Database languages — GQL." - International Organization for Standardization. diff --git a/lithoglyph/analytics/.editorconfig b/lithoglyph/analytics/.editorconfig deleted file mode 100644 index fc6650ce..00000000 --- a/lithoglyph/analytics/.editorconfig +++ /dev/null @@ -1,68 +0,0 @@ -# RSR-template-repo - Editor Configuration -# https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 2 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -trim_trailing_whitespace = false - -[*.adoc] -trim_trailing_whitespace = false - -[*.rs] -indent_size = 4 - -[*.ex] -indent_size = 2 - -[*.exs] -indent_size = 2 - -[*.zig] -indent_size = 4 - -[*.ada] -indent_size = 3 - -[*.adb] -indent_size = 3 - -[*.ads] -indent_size = 3 - -[*.hs] -indent_size = 2 - -[*.res] -indent_size = 2 - -[*.resi] -indent_size = 2 - -[*.ncl] -indent_size = 2 - -[*.rkt] -indent_size = 2 - -[*.scm] -indent_size = 2 - -[*.nix] -indent_size = 2 - -[Justfile] -indent_style = space -indent_size = 4 - -[justfile] -indent_style = space -indent_size = 4 diff --git a/lithoglyph/analytics/.gitattributes b/lithoglyph/analytics/.gitattributes deleted file mode 100644 index e860a85c..00000000 --- a/lithoglyph/analytics/.gitattributes +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitattributes - -* text=auto eol=lf - -# Source -*.rs text eol=lf diff=rust -*.ex text eol=lf diff=elixir -*.exs text eol=lf diff=elixir -*.jl text eol=lf -*.res text eol=lf -*.resi text eol=lf -*.ada text eol=lf diff=ada -*.adb text eol=lf diff=ada -*.ads text eol=lf diff=ada -*.hs text eol=lf -*.chpl text eol=lf -*.scm text eol=lf -*.ncl text eol=lf -*.nix text eol=lf - -# Docs -*.md text eol=lf diff=markdown -*.adoc text eol=lf -*.txt text eol=lf - -# Data -*.json text eol=lf -*.yaml text eol=lf -*.yml text eol=lf -*.toml text eol=lf - -# Config -.gitignore text eol=lf -.gitattributes text eol=lf -justfile text eol=lf -Makefile text eol=lf -Containerfile text eol=lf - -# Scripts -*.sh text eol=lf - -# Binary -*.png binary -*.jpg binary -*.gif binary -*.pdf binary -*.woff2 binary -*.zip binary -*.gz binary - -# Lock files -Cargo.lock text eol=lf -diff -flake.lock text eol=lf -diff diff --git a/lithoglyph/analytics/.gitignore b/lithoglyph/analytics/.gitignore deleted file mode 100644 index c7e0902a..00000000 --- a/lithoglyph/analytics/.gitignore +++ /dev/null @@ -1,34 +0,0 @@ -# Julia artifacts -Manifest.toml - -# IDE -.idea/ -.vscode/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Local config (may contain secrets) -config.toml -config.local.toml - -# Data directory (contains synced Parquet files) -data/ - -# Test artifacts -coverage/ -*.cov -target/ -node_modules/ -_build/ -deps/ -.elixir_ls/ -.cache/ -*.log -*.tmp -build/ -dist/ diff --git a/lithoglyph/analytics/ABI-FFI-README.md b/lithoglyph/analytics/ABI-FFI-README.md deleted file mode 100644 index e6a32bbf..00000000 --- a/lithoglyph/analytics/ABI-FFI-README.md +++ /dev/null @@ -1,385 +0,0 @@ -{{~ Aditionally delete this line and fill out the template below ~}} - -# {{PROJECT}} ABI/FFI Documentation - -## Overview - -This library follows the **Hyperpolymath RSR Standard** for ABI and FFI design: - -- **ABI (Application Binary Interface)** defined in **Idris2** with formal proofs -- **FFI (Foreign Function Interface)** implemented in **Zig** for C compatibility -- **Generated C headers** bridge Idris2 ABI to Zig FFI -- **Any language** can call through standard C ABI - -## Architecture - -``` -┌─────────────────────────────────────────────┐ -│ ABI Definitions (Idris2) │ -│ src/abi/ │ -│ - Types.idr (Type definitions) │ -│ - Layout.idr (Memory layout proofs) │ -│ - Foreign.idr (FFI declarations) │ -└─────────────────┬───────────────────────────┘ - │ - │ generates (at compile time) - ▼ -┌─────────────────────────────────────────────┐ -│ C Headers (auto-generated) │ -│ generated/abi/{{project}}.h │ -└─────────────────┬───────────────────────────┘ - │ - │ imported by - ▼ -┌─────────────────────────────────────────────┐ -│ FFI Implementation (Zig) │ -│ ffi/zig/src/main.zig │ -│ - Implements C-compatible functions │ -│ - Zero-cost abstractions │ -│ - Memory-safe by default │ -└─────────────────┬───────────────────────────┘ - │ - │ compiled to lib{{project}}.so/.a - ▼ -┌─────────────────────────────────────────────┐ -│ Any Language via C ABI │ -│ - Rust, ReScript, Julia, Python, etc. │ -└─────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -{{project}}/ -├── src/ -│ ├── abi/ # ABI definitions (Idris2) -│ │ ├── Types.idr # Core type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI function declarations -│ └── lib/ # Core library (any language) -│ -├── ffi/ -│ └── zig/ # FFI implementation (Zig) -│ ├── build.zig # Build configuration -│ ├── build.zig.zon # Dependencies -│ ├── src/ -│ │ └── main.zig # C-compatible FFI implementation -│ ├── test/ -│ │ └── integration_test.zig -│ └── include/ -│ └── {{project}}.h # C header (optional, can be generated) -│ -├── generated/ # Auto-generated files -│ └── abi/ -│ └── {{project}}.h # Generated from Idris2 ABI -│ -└── bindings/ # Language-specific wrappers (optional) - ├── rust/ - ├── rescript/ - └── julia/ -``` - -## Why Idris2 for ABI? - -### 1. **Formal Verification** - -Idris2's dependent types allow proving properties about the ABI at compile-time: - -```idris --- Prove struct size is correct -public export -exampleStructSize : HasSize ExampleStruct 16 - --- Prove field alignment is correct -public export -fieldAligned : Divides 8 (offsetOf ExampleStruct.field) - --- Prove ABI is platform-compatible -public export -abiCompatible : Compatible (ABI 1) (ABI 2) -``` - -### 2. **Type Safety** - -Encode invariants that C/Zig cannot express: - -```idris --- Non-null pointer guaranteed at type level -data Handle : Type where - MkHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> Handle - --- Array with length proof -data Buffer : (n : Nat) -> Type where - MkBuffer : Vect n Byte -> Buffer n -``` - -### 3. **Platform Abstraction** - -Platform-specific types with compile-time selection: - -```idris -CInt : Platform -> Type -CInt Linux = Bits32 -CInt Windows = Bits32 - -CSize : Platform -> Type -CSize Linux = Bits64 -CSize Windows = Bits64 -``` - -### 4. **Safe Evolution** - -Prove that new ABI versions are backward-compatible: - -```idris --- Compiler enforces compatibility -abiUpgrade : ABI 1 -> ABI 2 -abiUpgrade old = MkABI2 { - -- Must preserve all v1 fields - v1_compat = old, - -- Can add new fields - new_features = defaults -} -``` - -## Why Zig for FFI? - -### 1. **C ABI Compatibility** - -Zig exports C-compatible functions naturally: - -```zig -export fn library_function(param: i32) i32 { - return param * 2; -} -``` - -### 2. **Memory Safety** - -Compile-time safety without runtime overhead: - -```zig -// Null check enforced at compile time -const handle = init() orelse return error.InitFailed; -defer free(handle); -``` - -### 3. **Cross-Compilation** - -Built-in cross-compilation to any platform: - -```bash -zig build -Dtarget=x86_64-linux -zig build -Dtarget=aarch64-macos -zig build -Dtarget=x86_64-windows -``` - -### 4. **Zero Dependencies** - -No runtime, no libc required (unless explicitly needed): - -```zig -// Minimal binary size -pub const lib = @import("std"); -// Only includes what you use -``` - -## Building - -### Build FFI Library - -```bash -cd ffi/zig -zig build # Build debug -zig build -Doptimize=ReleaseFast # Build optimized -zig build test # Run tests -``` - -### Generate C Header from Idris2 ABI - -```bash -cd src/abi -idris2 --cg c-header Types.idr -o ../../generated/abi/{{project}}.h -``` - -### Cross-Compile - -```bash -cd ffi/zig - -# Linux x86_64 -zig build -Dtarget=x86_64-linux - -# macOS ARM64 -zig build -Dtarget=aarch64-macos - -# Windows x86_64 -zig build -Dtarget=x86_64-windows -``` - -## Usage - -### From C - -```c -#include "{{project}}.h" - -int main() { - void* handle = {{project}}_init(); - if (!handle) return 1; - - int result = {{project}}_process(handle, 42); - if (result != 0) { - const char* err = {{project}}_last_error(); - fprintf(stderr, "Error: %s\n", err); - } - - {{project}}_free(handle); - return 0; -} -``` - -Compile with: -```bash -gcc -o example example.c -l{{project}} -L./zig-out/lib -``` - -### From Idris2 - -```idris -import {{PROJECT}}.ABI.Foreign - -main : IO () -main = do - Just handle <- init - | Nothing => putStrLn "Failed to initialize" - - Right result <- process handle 42 - | Left err => putStrLn $ "Error: " ++ errorDescription err - - free handle - putStrLn "Success" -``` - -### From Rust - -```rust -#[link(name = "{{project}}")] -extern "C" { - fn {{project}}_init() -> *mut std::ffi::c_void; - fn {{project}}_free(handle: *mut std::ffi::c_void); - fn {{project}}_process(handle: *mut std::ffi::c_void, input: u32) -> i32; -} - -fn main() { - unsafe { - let handle = {{project}}_init(); - assert!(!handle.is_null()); - - let result = {{project}}_process(handle, 42); - assert_eq!(result, 0); - - {{project}}_free(handle); - } -} -``` - -### From Julia - -```julia -const lib{{project}} = "lib{{project}}" - -function init() - handle = ccall((:{{project}}_init, lib{{project}}), Ptr{Cvoid}, ()) - handle == C_NULL && error("Failed to initialize") - handle -end - -function process(handle, input) - result = ccall((:{{project}}_process, lib{{project}}), Cint, (Ptr{Cvoid}, UInt32), handle, input) - result -end - -function cleanup(handle) - ccall((:{{project}}_free, lib{{project}}), Cvoid, (Ptr{Cvoid},), handle) -end - -# Usage -handle = init() -try - result = process(handle, 42) - println("Result: $result") -finally - cleanup(handle) -end -``` - -## Testing - -### Unit Tests (Zig) - -```bash -cd ffi/zig -zig build test -``` - -### Integration Tests - -```bash -cd ffi/zig -zig build test-integration -``` - -### ABI Verification (Idris2) - -```idris --- Compile-time verification -%runElab verifyABI - --- Runtime checks -main : IO () -main = do - verifyLayoutsCorrect - verifyAlignmentsCorrect - putStrLn "ABI verification passed" -``` - -## Contributing - -When modifying the ABI/FFI: - -1. **Update ABI first** (`src/abi/*.idr`) - - Modify type definitions - - Update proofs - - Ensure backward compatibility - -2. **Generate C header** - ```bash - idris2 --cg c-header src/abi/Types.idr -o generated/abi/{{project}}.h - ``` - -3. **Update FFI implementation** (`ffi/zig/src/main.zig`) - - Implement new functions - - Match ABI types exactly - -4. **Add tests** - - Unit tests in Zig - - Integration tests - - ABI verification tests - -5. **Update documentation** - - Function signatures - - Usage examples - - Migration guide (if breaking changes) - -## License - -PMPL-1.0-or-later - -## See Also - -- [Idris2 Documentation](https://idris2.readthedocs.io) -- [Zig Documentation](https://ziglang.org/documentation/master/) -- [Rhodium Standard Repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [FFI Migration Guide](../ffi-migration-guide.md) -- [ABI Migration Guide](../abi-migration-guide.md) diff --git a/lithoglyph/analytics/LICENSE b/lithoglyph/analytics/LICENSE deleted file mode 100644 index ec540b34..00000000 --- a/lithoglyph/analytics/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: MPL-2.0 -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/lithoglyph/analytics/MAINTAINERS.adoc b/lithoglyph/analytics/MAINTAINERS.adoc deleted file mode 100644 index 48d97817..00000000 --- a/lithoglyph/analytics/MAINTAINERS.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Maintainers -:toc: preamble - -This document lists the maintainers of this project and their responsibilities. - -== Current Maintainers - -[cols="2,3,2",options="header"] -|=== -| Name | Role | Contact - -| Jonathan D.A. Jewell -| Lead Maintainer -| https://github.com/hyperpolymath[@hyperpolymath] -|=== - -== Responsibilities - -Maintainers are responsible for: - -* Reviewing and merging pull requests -* Triaging issues and feature requests -* Ensuring code quality and security standards -* Managing releases and versioning -* Upholding the project's code of conduct - -== Becoming a Maintainer - -Contributors who demonstrate: - -* Consistent, high-quality contributions -* Understanding of the project's goals and standards -* Constructive participation in discussions -* Commitment to the project's long-term health - -May be invited to become maintainers at the discretion of existing maintainers. - -== Decision Making - -* Routine decisions (bug fixes, minor improvements) can be made by any maintainer -* Significant changes require discussion and consensus among maintainers -* Breaking changes or major features should be discussed in issues before implementation - -== Contact - -For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/lithoglyph/analytics/MOVED.md b/lithoglyph/analytics/MOVED.md deleted file mode 100644 index 7716257d..00000000 --- a/lithoglyph/analytics/MOVED.md +++ /dev/null @@ -1,12 +0,0 @@ -# This Repository Has Moved - -**This component is now part of the [Lithoglyph monorepo](https://github.com/hyperpolymath/lith).** - -## New Location - -- **Monorepo:** https://github.com/hyperpolymath/lith -- **This Component:** https://github.com/hyperpolymath/lith/tree/main/analyticsanalyticsanalyticsextensions/analytics - -All Lithoglyph development now happens in the monorepo. - -**See you at [github.com/hyperpolymath/lith](https://github.com/hyperpolymath/lith)!** 🚀 diff --git a/lithoglyph/analytics/Project.toml b/lithoglyph/analytics/Project.toml deleted file mode 100644 index 8ae3e0ad..00000000 --- a/lithoglyph/analytics/Project.toml +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name = "LithAnalytics" -uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" -authors = ["Hyperpolymath "] -version = "0.1.0" - -[deps] -Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" -CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" -JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" -Parquet2 = "98572fba-bba0-415d-956f-fa77e587d26d" -TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[targets] -test = ["Test"] - -[compat] -julia = "1.10" diff --git a/lithoglyph/analytics/README.adoc b/lithoglyph/analytics/README.adoc deleted file mode 100644 index 1e73f38b..00000000 --- a/lithoglyph/analytics/README.adoc +++ /dev/null @@ -1,179 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith-Analytics -:toc: macro -:toc-title: Contents -:toclevels: 3 - -OLAP analytics layer for Lith - columnar aggregations and time-series analysis. - -== Overview - -Lith-Analytics provides high-performance analytical queries over Lith documents. -While Lith prioritizes auditability and reversibility, analytics workloads require -different optimization strategies. This separation allows Lith to maintain its -principles while enabling fast aggregations, rollups, and time-series analysis. - -== Architecture - -[source] ----- -┌─────────────────────────────────────────────────────────────┐ -│ Lith (Source of Truth) │ -│ Documents with PROMPT scores │ -└─────────────────────────────────┬───────────────────────────┘ - │ HTTP API - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Lith-Analytics │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Ingester │ │ Columnar │ │ Query │ │ -│ │ │──▶│ Store │──▶│ Engine │ │ -│ │ (ETL from │ │ (Arrow/ │ │ (DataFrames, │ │ -│ │ Lith) │ │ Parquet) │ │ Aggregates) │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ HTTP API -┌─────────────────────────────────────────────────────────────┐ -│ Consumers (lithoglyph-studio, dashboards, reports) │ -└─────────────────────────────────────────────────────────────┘ ----- - -== Key Features - -* **Columnar Storage**: Arrow/Parquet for analytical query patterns -* **PROMPT Score Analytics**: Aggregations over epistemological dimensions -* **Time-Series Analysis**: Document creation/modification trends -* **Rollups**: Pre-computed aggregations for common queries -* **Provenance Tracking**: Analyze who contributed what, when - -== Why Julia? - -Lith-Analytics uses Julia because: - -1. **Native columnar operations**: DataFrames.jl is optimized for analytical workloads -2. **Arrow integration**: Arrow.jl provides zero-copy interop -3. **Performance**: JIT compilation approaches C performance -4. **Scientific computing**: Strong ecosystem for statistical analysis -5. **Hyperpolymath policy**: Julia is the approved language for data/batch processing - -== API Endpoints - -=== Analytics Queries - -[source] ----- -GET /analytics/health - Health check - -GET /analytics/stats - Overall statistics about indexed data - -POST /analytics/query - Execute analytical query - Body: { "query": "...", "params": {...} } - -GET /analytics/prompt-scores?collection=X&groupBy=Y - PROMPT score aggregations - -GET /analytics/time-series?collection=X&field=Y&interval=day - Time-series analysis - -GET /analytics/contributors?collection=X - Contributor/provenance analysis ----- - -=== Data Management - -[source] ----- -POST /analytics/sync - Sync data from Lith (incremental or full) - Body: { "collection": "...", "mode": "incremental|full" } - -GET /analytics/collections - List synced collections with stats ----- - -== Configuration - -[source,toml] ----- -[lithoglyph] -api_url = "http://localhost:8080" -collections = ["evidence", "claims"] - -[server] -host = "127.0.0.1" -port = 8082 - -[storage] -# Path for Parquet files -data_dir = "./data" -# Retention in days (0 = forever) -retention_days = 0 - -[sync] -# Auto-sync interval in minutes (0 = manual only) -auto_sync_minutes = 60 ----- - -== PROMPT Score Analytics - -Lith documents may include PROMPT epistemological scores: - -* **P**rovenance - Source traceability -* **R**eplicability - Can findings be reproduced? -* **O**bjective - Methodological rigor -* **M**ethodology - Analytical approach quality -* **P**ublication - Peer review status -* **T**ransparency - Data/method openness - -Lith-Analytics provides aggregations: - -[source,julia] ----- -# Average PROMPT scores by collection -prompt_stats(collection="evidence", groupby=:source) - -# Score distribution histograms -prompt_distribution(collection="evidence", dimension=:provenance) - -# Correlation between dimensions -prompt_correlations(collection="evidence") ----- - -== Development - -=== Prerequisites - -* Julia 1.10+ -* Lith instance running - -=== Setup - -[source,bash] ----- -cd lithoglyph-analytics -julia --project=. -e 'using Pkg; Pkg.instantiate()' ----- - -=== Running - -[source,bash] ----- -julia --project=. src/main.jl --config config.toml ----- - -=== Testing - -[source,bash] ----- -julia --project=. -e 'using Pkg; Pkg.test()' ----- - -== License - -PMPL-1.0-or-later - -Part of the Lith ecosystem. diff --git a/lithoglyph/analytics/README.md b/lithoglyph/analytics/README.md deleted file mode 100644 index 9d6302f9..00000000 --- a/lithoglyph/analytics/README.md +++ /dev/null @@ -1,270 +0,0 @@ -# Lith Advanced Analytics - -Advanced analytics features for Lith including aggregations, time series analysis, window functions, and visualization exports. - -## Features - -| Feature | File | Description | -|---------|------|-------------| -| Aggregations | `Lith_Analytics_Aggregations.res` | Statistical aggregations | -| Time Series | `Lith_Analytics_TimeSeries.res` | Time-based analysis | -| Window Functions | `Lith_Analytics_Window.res` | SQL-style windows | -| Export | `Lith_Analytics_Export.res` | Visualization exports | - -## Aggregations - -Statistical aggregation functions for data analysis. - -```rescript -// Basic aggregations -let values = [1.0, 2.0, 3.0, 4.0, 5.0] -let total = sum(values) // 15.0 -let average = avg(values) // 3.0 -let minimum = minValue(values) // Some(1.0) -let maximum = maxValue(values) // Some(5.0) -let mid = median(values) // Some(3.0) -let sd = stddev(values) // 1.414... -let p95 = percentile(values, 95.0) // 4.8 - -// Group by with aggregations -let results = groupBy( - documents, - ["category", "region"], - [(Sum, "amount"), (Avg, "price"), (Count, "id")] -) -``` - -### Supported Aggregations - -| Function | Description | -|----------|-------------| -| `Count` | Count of values | -| `Sum` | Sum of numeric values | -| `Avg` | Arithmetic mean | -| `Min` | Minimum value | -| `Max` | Maximum value | -| `Median` | Median (50th percentile) | -| `Stddev` | Standard deviation | -| `Variance` | Variance | -| `Percentile(n)` | nth percentile | -| `CountDistinct` | Count of unique values | - -## Time Series Analysis - -Time-based data analysis and trend detection. - -```rescript -// Create time series -let points = [ - {timestamp: 1704067200000.0, value: 100.0, metadata: None}, - {timestamp: 1704153600000.0, value: 110.0, metadata: None}, - {timestamp: 1704240000000.0, value: 105.0, metadata: None}, -] - -let series = createTimeSeries("sales", "amount", points, Day) - -// Analyze trend -let trend = analyzeTrend(series) -// {direction: Increasing, slope: 2.5, rSquared: 0.92, changePercent: 5.0} - -// Moving average -let ma = movingAverage(series, 7) // 7-day moving average - -// Anomaly detection -let anomalies = detectAnomalies(series, 2.0) // 2 standard deviations -``` - -### Time Granularities - -| Granularity | Description | -|-------------|-------------| -| `Second` | Per-second buckets | -| `Minute` | Per-minute buckets | -| `Hour` | Per-hour buckets | -| `Day` | Per-day buckets | -| `Week` | Per-week buckets | -| `Month` | Per-month buckets | -| `Quarter` | Per-quarter buckets | -| `Year` | Per-year buckets | - -### Trend Directions - -| Direction | Meaning | -|-----------|---------| -| `Increasing` | Positive trend (R² > 0.3, slope > 0.01) | -| `Decreasing` | Negative trend (R² > 0.3, slope < -0.01) | -| `Stable` | No significant trend | -| `Volatile` | High variance (R² < 0.3) | - -## Window Functions - -SQL-style window functions for advanced analytics. - -```rescript -// Define window specification -let spec = { - partitionBy: ["department"], - orderBy: [("salary", false)], // descending - frameType: Rows, - frameStart: UnboundedPreceding, - frameEnd: CurrentRow, -} - -// Apply window function -let results = execute(employees, Rank, spec) -// Returns rank within each department - -// Cumulative sum -let cumSum = execute(sales, CumulativeSum("amount"), { - ...defaultWindowSpec, - orderBy: [("date", true)], -}) - -// Lead/Lag for comparison -let yoyGrowth = execute(revenue, Lag("amount", 12, Some(0.0)), { - ...defaultWindowSpec, - orderBy: [("month", true)], -}) -``` - -### Available Window Functions - -| Function | Description | -|----------|-------------| -| `RowNumber` | Sequential row number | -| `Rank` | Rank with gaps | -| `DenseRank` | Rank without gaps | -| `Ntile(n)` | Divide into n buckets | -| `Lag(field, n, default)` | Value n rows before | -| `Lead(field, n, default)` | Value n rows after | -| `FirstValue(field)` | First value in frame | -| `LastValue(field)` | Last value in frame | -| `NthValue(field, n)` | nth value in frame | -| `CumulativeSum(field)` | Running total | -| `RunningAvg(field)` | Running average | -| `PercentRank` | Relative rank (0-1) | - -## Export - -Data export for visualization and reporting. - -```rescript -// Export to CSV -let csv = export(data, {...defaultOptions, format: CSV}) - -// Export for Chart.js -let chartConfig = { - chartType: Line, - title: "Monthly Sales", - xAxis: "month", - yAxis: "amount", - series: ["actual", "forecast"], - colors: Some(["#3b82f6", "#ef4444"]), - legend: true, -} -let chartData = export(data, { - ...defaultOptions, - format: ChartJS, - chartConfig: Some(chartConfig), -}) - -// Export for Vega-Lite -let vegaSpec = export(data, { - ...defaultOptions, - format: Vega, - chartConfig: Some({...chartConfig, chartType: Bar}), -}) - -// Export as table data -let tableData = export(data, {...defaultOptions, format: TableData}) -``` - -### Export Formats - -| Format | Description | Use Case | -|--------|-------------|----------| -| `CSV` | Comma-separated values | Spreadsheets | -| `JSON` | JSON array | APIs | -| `ChartJS` | Chart.js config | Web charts | -| `D3` | D3-compatible data | Custom viz | -| `Vega` | Vega-Lite spec | Declarative charts | -| `TableData` | Columns + rows | Data tables | - -### Chart Types - -| Type | Description | -|------|-------------| -| `Line` | Line chart | -| `Bar` | Bar chart | -| `Pie` | Pie chart | -| `Scatter` | Scatter plot | -| `Area` | Area chart | -| `Histogram` | Histogram | -| `Heatmap` | Heatmap | - -## Architecture - -``` -analytics/ -├── README.md -└── src/ - ├── Lith_Analytics_Aggregations.res # Statistical aggregations - ├── Lith_Analytics_TimeSeries.res # Time series analysis - ├── Lith_Analytics_Window.res # Window functions - └── Lith_Analytics_Export.res # Visualization exports -``` - -## Use Cases - -### Sales Analytics - -```rescript -// Monthly sales by region -let monthlySales = groupBy( - orders, - ["region", "month"], - [(Sum, "amount"), (Count, "id"), (Avg, "amount")] -) - -// Sales trend -let series = createTimeSeries("sales", "amount", salesData, Month) -let trend = analyzeTrend(series) -``` - -### Financial Analysis - -```rescript -// Year-over-year comparison -let yoyResults = execute(revenue, Lag("amount", 12, None), { - partitionBy: ["product"], - orderBy: [("month", true)], - frameType: Rows, - frameStart: UnboundedPreceding, - frameEnd: CurrentRow, -}) - -// Moving average for smoothing -let ma20 = movingAverage(stockPrices, 20) -``` - -### User Analytics - -```rescript -// Cohort analysis -let cohorts = groupBy( - users, - ["signup_month", "activity_month"], - [(CountDistinct, "user_id")] -) - -// Percentile rankings -let userRanks = execute(users, PercentRank, { - partitionBy: ["segment"], - orderBy: [("engagement_score", false)], - ...defaultWindowSpec, -}) -``` - -## License - -PMPL-1.0-or-later diff --git a/lithoglyph/analytics/ROADMAP.adoc b/lithoglyph/analytics/ROADMAP.adoc deleted file mode 100644 index a78e0ed9..00000000 --- a/lithoglyph/analytics/ROADMAP.adoc +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Analytics Roadmap - -== Current Status - -Initial development phase. - -== Milestones - -=== v0.1.0 - Foundation -* [ ] Core functionality -* [ ] Basic documentation -* [ ] CI/CD pipeline - -=== v1.0.0 - Stable Release -* [ ] Full feature set -* [ ] Comprehensive tests -* [ ] Production ready - -== Future Directions - -_To be determined based on community feedback._ diff --git a/lithoglyph/analytics/RSR_OUTLINE.adoc b/lithoglyph/analytics/RSR_OUTLINE.adoc deleted file mode 100644 index ede631fe..00000000 --- a/lithoglyph/analytics/RSR_OUTLINE.adoc +++ /dev/null @@ -1,218 +0,0 @@ -= RSR Template Repository - -image:[Palimpsest-MPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] image:[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] -:toc: -:sectnums: - -// Badges -image:https://img.shields.io/badge/RSR-Infrastructure-cd7f32[RSR Infrastructure] -image:https://img.shields.io/badge/Phase-Maintenance-brightgreen[Phase] -image:https://img.shields.io/badge/Guix-Primary-purple?logo=gnu[Guix] - -== Overview - -**The canonical template for RSR (Rhodium Standard Repository) projects.** - -This repository provides the standardized structure, configuration, and tooling for all 139 repos in the hyperpolymath ecosystem. Use it to: - -* Bootstrap new projects with RSR compliance -* Reference the standard directory structure -* Copy configuration templates (Justfile, STATE.scm, etc.) - -== Quick Start - -[source,bash] ----- -# Clone the template -git clone https://github.com/hyperpolymath/RSR-template-repo my-project -cd my-project - -# Remove template git history -rm -rf .git -git init - -# Customize -sed -i 's/RSR-template-repo/my-project/g' Justfile guix.scm README.adoc - -# Enter development environment -guix shell -D -f guix.scm - -# Validate compliance -just validate-rsr ----- - -== What's Included - -[cols="1,3"] -|=== -|File/Directory |Purpose - -|`.editorconfig` -|Editor configuration (indent, charset) - -|`.gitignore` -|Standard ignore patterns - -|`.guix-channel` -|Guix channel definition - -|`.well-known/` -|RFC-compliant metadata (security.txt, ai.txt, humans.txt) - -|`docs/` -|Documentation directory - -|`guix.scm` -|Guix package definition - -|`justfile` -|Task runner with 50+ recipes - -|`LICENSE.txt` -|Palimpsest (PMPL-1.0-or-later) - -|`README.adoc` -|This file - -|`RSR_COMPLIANCE.adoc` -|Compliance tracking - -|`STATE.scm` -|Project state checkpoint -|=== - -== Justfile Features - -The template Justfile provides: - -* **~10 billion recipe combinations** via matrix recipes -* **Cookbook generation**: `just cookbook` → `docs/just-cookbook.adoc` -* **Man page generation**: `just man` → `docs/man/project.1` -* **RSR validation**: `just validate-rsr` -* **STATE.scm management**: `just state-touch`, `just state-phase` -* **Container support**: `just container-build`, `just container-push` -* **CI matrix**: `just ci-matrix [stage] [depth]` - -=== Key Recipes - -[source,bash] ----- -just # Show all recipes -just help # Detailed help -just info # Project info -just combinations # Show matrix options - -just build # Build (debug) -just test # Run tests -just quality # Format + lint + test -just ci # Full CI pipeline - -just validate # RSR + STATE validation -just docs # Generate all docs -just cookbook # Generate Justfile docs - -just guix-shell # Guix dev environment -just container-build # Build container ----- - -== Directory Structure - -[source] ----- -project/ -├── .editorconfig # Editor settings -├── .gitignore # Git ignore -├── .guix-channel # Guix channel -├── .well-known/ # RFC metadata -│ ├── ai.txt -│ ├── humans.txt -│ └── security.txt -├── config/ # Nickel configs (optional) -├── docs/ # Documentation -│ ├── generated/ -│ ├── man/ -│ └── just-cookbook.adoc -├── guix.scm # Guix package -├── Justfile # Task runner -├── LICENSE.txt # Dual license -├── README.adoc # Overview -├── RSR_COMPLIANCE.adoc # Compliance -├── src/ # Source code -├── STATE.scm # State checkpoint -└── tests/ # Tests ----- - -== RSR Compliance - -=== Language Tiers - -* **Tier 1** (Gold): Rust, Elixir, Zig, Ada, Haskell, ReScript -* **Tier 2** (Silver): Nickel, Racket, Guile Scheme, Nix -* **Infrastructure**: Guix channels, derivations - -=== Required Files - -* `.editorconfig` -* `.gitignore` -* `justfile` -* `README.adoc` -* `RSR_COMPLIANCE.adoc` -* `LICENSE` (PMPL-1.0-or-later) -* `.well-known/security.txt` -* `.well-known/ai.txt` -* `.well-known/humans.txt` -* `guix.scm` OR `flake.nix` - -=== Prohibited - -* Python outside `salt/` directory -* TypeScript/JavaScript (use ReScript) -* CUE (use Guile/Nickel) -* `Dockerfile` (use `Containerfile`) - -== STATE.scm - -The STATE.scm file tracks project state: - -[source,scheme] ----- -(define state - `((metadata - (project . "my-project") - (updated . "2025-12-10")) - (position - (phase . implementation) ; design|implementation|testing|maintenance|archived - (maturity . beta)) ; experimental|alpha|beta|production|lts - (ecosystem - (part-of . ("RSR Framework")) - (depends-on . ())))) ----- - -== Badge Schema - -Generate badges from STATE.scm: - -[source,bash] ----- -just badges standard ----- - -See `docs/BADGE_SCHEMA.adoc` for the full badge taxonomy. - -== Ecosystem Integration - -This template is part of: - -* **STATE.scm Ecosystem**: Conversation checkpoints -* **RSR Framework**: Repository standards -* **Consent-Aware-HTTP**: .well-known compliance - -== License - -SPDX-License-Identifier: CC-BY-SA-4.0 - -== Links - -* https://github.com/hyperpolymath/elegant-STATE[elegant-STATE] - STATE.scm tooling -* https://github.com/hyperpolymath/conative-gating[conative-gating] - Policy enforcement -* https://rhodium.sh[Rhodium Standard] - RSR documentation diff --git a/lithoglyph/analytics/UNIFIED-ROADMAP.scm b/lithoglyph/analytics/UNIFIED-ROADMAP.scm deleted file mode 100644 index 3e5b6efe..00000000 --- a/lithoglyph/analytics/UNIFIED-ROADMAP.scm +++ /dev/null @@ -1,248 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith Ecosystem - Unified Roadmap to MVP 1.0.0 -; Media-Type: application/vnd.roadmap+scm -; -; This file is distributed to all Lith ecosystem repos: -; - lith (core database) -; - gql-dt (dependently-typed query language) -; - lith-studio (GUI) -; - lith-debugger (recovery tool) -; - lith-geo (geospatial projection layer) -; - lith-analytics (OLAP analytics projection layer) - -(unified-roadmap - (metadata - (version "1.1.0") - (created "2026-01-12") - (updated "2026-01-16") - (author "hyperpolymath") - (target "MVP 1.0.0")) - - ;; ============================================================================ - ;; ECOSYSTEM OVERVIEW - ;; ============================================================================ - (ecosystem-summary - (components - (lith - (version "0.0.5") - (completion 80) - (role "Core database engine") - (tech "Forth + Factor + Zig")) - (gql-dt - (version "0.2.0") - (completion 65) - (role "Dependently-typed query language") - (tech "Lean 4 + Zig")) - (lith-studio - (version "0.1.0") - (completion 45) - (role "Zero-friction GUI") - (tech "ReScript + Tauri 2.0 + Rust")) - (lith-debugger - (version "0.1.0") - (completion 55) - (role "Proof-carrying recovery tool") - (tech "Lean 4 + Idris 2 + Rust")) - (lith-geo - (version "0.1.0") - (completion 15) - (role "Geospatial projection layer") - (tech "Rust (rstar, axum, geo)")) - (lith-analytics - (version "0.1.0") - (completion 15) - (role "OLAP analytics projection layer") - (tech "Julia (DataFrames, Parquet2, Oxygen)"))) - - (architecture - "┌─────────────────────────────────────────────────────────────┐" - "│ Lith Studio (GUI) │" - "│ ↓ generates FQLdt code │" - "├─────────────────────────────────────────────────────────────┤" - "│ FQLdt (Lean 4) │" - "│ ↓ compiles to proof blobs │" - "├─────────────────────────────────────────────────────────────┤" - "│ Form.Bridge (Zig ABI) │" - "│ ↓ calls │" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Core (Forth + Factor) │" - "│ Form.Runtime → Form.Normalizer → Form.Model → Form.Blocks│" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Debugger (alongside) │" - "│ ↓ proves recovery safe │" - "│ Lith + FQLdt │" - "├─────────────────────────────────────────────────────────────┤" - "│ Projection Layers (read from Lith HTTP API) │" - "│ lith-geo (R-tree spatial) lith-analytics (OLAP) │" - "└─────────────────────────────────────────────────────────────┘")) - - ;; ============================================================================ - ;; CRITICAL PATH TO MVP 1.0.0 - ;; ============================================================================ - (critical-path - (phase (id "P1") (name "Core Integration") - (duration "weeks 1-6") - (focus "Lith + FQLdt integration") - - (lith-tasks - (task "Complete M11: HTTP API Server" priority: critical status: complete) - (task "M12: Language bindings (ReScript, PHP)" priority: critical status: next) - (task "M13: CMS integration (WordPress)" priority: high status: pending) - (task "M14: Form.ControlPlane (clustering)" priority: medium status: pending)) - - (gql-dt-tasks - (task "M5: Zig FFI bridge to Form.Bridge" priority: critical status: not-started) - (task "M6: GQL parser (integrate with Lith's EBNF)" priority: high status: not-started) - (task "Proof blob serialization (CBOR RFC 8949)" priority: high status: pending)) - - (checkpoint "FQLdt can compile a query → proof blob → Lith accepts and executes")) - - (phase (id "P2") (name "User-Facing Tools") - (duration "weeks 7-10") - (focus "Studio and Debugger completion") - - (studio-tasks - (task "Verify ReScript/Tauri build pipeline" priority: critical status: pending) - (task "Wire ReScript UI to FQLdt code generation" priority: high status: pending) - (task "Connect to Lith HTTP API" priority: high status: blocked) - (task "Test schema creation → query → results flow" priority: medium status: pending)) - - (debugger-tasks - (task "Wire Idris REPL to PostgreSQL adapter" priority: high status: pending) - (task "Lith adapter: parse real journal files" priority: high status: partial) - (task "Complete Ratatui TUI interface" priority: medium status: in-progress) - (task "Integration: proof verification before recovery" priority: medium status: pending)) - - (projection-layer-tasks - (task "lith-geo: Integration test with real Lith" priority: high status: pending) - (task "lith-geo: Docker deployment" priority: medium status: pending) - (task "lith-analytics: Integration test with real Lith" priority: high status: pending) - (task "lith-analytics: PROMPT score dashboard endpoints" priority: medium status: pending)) - - (checkpoint "Users can create schemas in Studio, debug with Debugger, query spatial/analytics")) - - (phase (id "P3") (name "Production Hardening") - (duration "weeks 11-12") - (focus "Stability and polish") - - (all-repos - (task "Crash recovery tests" priority: high) - (task "Error handling improvements" priority: high) - (task "Cross-platform testing" priority: medium) - (task "Documentation completion" priority: medium) - (task "Performance optimization" priority: low)) - - (checkpoint "MVP 1.0.0 release ready"))) - - ;; ============================================================================ - ;; DEPENDENCY GRAPH - ;; ============================================================================ - (dependencies - (lith-m11 - (name "Lith HTTP API Server") - (blocks "Studio M2" "Debugger Lith adapter") - (priority critical)) - - (gql-dt-m5 - (name "FQLdt Zig FFI Bridge") - (blocks "Studio M3" "Real type checking") - (depends-on "Lith Form.Bridge") - (priority critical)) - - (gql-dt-m6 - (name "FQLdt GQL Parser") - (blocks "Full FQLdt compilation") - (depends-on "gql-dt-m5") - (priority high)) - - (studio-m1 - (name "Studio Build Pipeline") - (blocks "All Studio features") - (priority critical)) - - (debugger-repl-db - (name "Debugger REPL Database Connection") - (blocks "Real debugging") - (priority high)) - - (lith-geo-integration - (name "lith-geo Lith Integration") - (depends-on "lith-m11") - (blocks "Spatial queries in Studio") - (priority medium)) - - (lith-analytics-integration - (name "lith-analytics Lith Integration") - (depends-on "lith-m11") - (blocks "Analytics dashboards in Studio") - (priority medium))) - - ;; ============================================================================ - ;; UNRESOLVED DECISIONS - ;; ============================================================================ - (decisions-needed - (decision (id "DECISION-002") - (title "FQLdt parser approach") - (repo "gql-dt") - (options - "Hand-rolled parser (simple, no deps)" - "Lean 4 Parsec (built-in)" - "Integrate with Lith's Factor-based GQL parser") - (recommendation "Integrate - reuse Lith's EBNF grammar via FFI") - (impact "Affects M6 implementation")) - - (decision (id "DECISION-003") - (title "Lith integration strategy for FQLdt") - (repo "gql-dt") - (options - "Mock Forth core for MVP" - "Real Form.Bridge integration") - (recommendation "Real integration - M11 HTTP API makes this feasible") - (impact "Determines MVP scope"))) - - ;; ============================================================================ - ;; POST-MVP ROADMAP - ;; ============================================================================ - (post-mvp - (release (version "1.1.0") (name "Normalization & Migration") - (features - "Form.Normalizer full integration (FD discovery → decomposition)" - "Three-phase migration workflow (Announce/Shadow/Commit)" - "Studio: visual normalization wizard" - "Debugger: migration rollback proofs" - "lith-geo: Polygon and region queries" - "lith-analytics: Time-series dashboards in Studio")) - - (release (version "1.2.0") (name "Multi-Database Support") - (features - "Debugger: SQLite adapter completion" - "Lith: clustering/replication (Form.ControlPlane begins)" - "Studio: connection manager for multiple DBs")) - - (release (version "2.0.0") (name "Agentic Ecosystem") - (features - "Form.ControlPlane (Elixir/OTP) for distributed coordination" - "Agent handover protocols" - "Long-term archive format standardization" - "Multi-user collaboration in Studio"))) - - ;; ============================================================================ - ;; SUCCESS METRICS - ;; ============================================================================ - (success-metrics - (mvp-criteria - "User can create a schema in Studio with visual builder" - "Schema generates valid FQLdt with type checking" - "User can insert data with provenance tracking" - "User can query data and see results" - "Debugger can analyze schema and propose fixes" - "All operations have proof-carrying verification") - - (quality-gates - "All ReScript code compiles without warnings" - "All Rust code passes Clippy lints" - "All Lean 4 code builds with lake" - "All Julia code passes tests" - "Cross-platform builds succeed (Mac/Windows/Linux)" - "Integration tests pass end-to-end" - "Projection layers can sync from Lith HTTP API"))) diff --git a/lithoglyph/analytics/config.example.toml b/lithoglyph/analytics/config.example.toml deleted file mode 100644 index 38f88eba..00000000 --- a/lithoglyph/analytics/config.example.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Lith-Analytics Configuration Example -# Copy to config.toml and adjust as needed - -[lithoglyph] -# Lith HTTP API URL -api_url = "http://localhost:8080" -# Collections to sync for analytics -collections = ["evidence", "claims"] - -[server] -# Host to bind to -host = "127.0.0.1" -# Port to listen on -port = 8082 - -[storage] -# Directory for Parquet data files -data_dir = "./data" -# Retention in days (0 = forever) -retention_days = 0 - -[sync] -# Auto-sync interval in minutes (0 = manual only via POST /analytics/sync) -auto_sync_minutes = 60 diff --git a/lithoglyph/analytics/docs/CITATIONS.adoc b/lithoglyph/analytics/docs/CITATIONS.adoc deleted file mode 100644 index 6f167bdf..00000000 --- a/lithoglyph/analytics/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {PMPL-1.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/lithoglyph/analytics/examples/SafeDOMExample.affine b/lithoglyph/analytics/examples/SafeDOMExample.affine deleted file mode 100644 index 2a62c1d1..00000000 --- a/lithoglyph/analytics/examples/SafeDOMExample.affine +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SafeDOMExample.affine — formally-verified DOM mounting (aspirational). -// -// This example shows the *shape* of SafeDOM consumer code in current -// AffineScript syntax. The `SafeDOM` stdlib surface it references -// (`mount_safe`, `mount_when_ready`, `mount_batch`, -// `proven_selector_validate`, `proven_html_validate`, `mount`) is the -// target of `affinescript#56` (DOM+Pixi binding survey) and does not -// yet exist in the published stdlib. The file is therefore -// parse-checked but not type-checked end-to-end until #56 lands the -// bindings; `affinescript check` reports `Resolve.UndefinedModule -// SafeDOM` which is expected. -// -// Previous versions of this file (estate-wide, 5 dialect variants) -// pre-dated ADR-014 (qualified paths), ADR-016 (effect rows), and the -// `#{`-record-literal sigil (ADR-215). They were retired in favour of -// this canonical via the gitbot-fleet#208 sweep (2026-05-26). - -module SafeDOMExample; - -use prelude::{Option, Some, None, Result, Ok, Err}; - -// `Element` and friends are nominal extern types for now — the real -// shape lands with affinescript#56. -extern type Element; -extern type Selector; -extern type ValidHTML; - -// Single-mount status, lifted from the host into a typed tag union. -enum MountStatus { - Mounted(Element), - MountPointNotFound(String), - InvalidSelector(String), - InvalidHTML(String) -} - -// Batch-mount result. -enum MountResult { - Mounted([Element]), - Failed(String) -} - -// Spec for one element in a batch mount. -struct MountSpec { - selector: String, - html: String -} - -// SafeDOM's host-side surface, all IO-effecting. Callbacks are passed -// as separate parameters (rather than a `MountCallbacks` record) -// because fn-typed struct fields are not currently parser-supported. -extern fn mount_safe( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_when_ready( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_batch(specs: ref [MountSpec]) -{IO}-> MountResult; - -extern fn proven_selector_validate(s: ref String) -{IO}-> Result; -extern fn proven_html_validate(s: ref String) -{IO}-> Result; -extern fn mount(sel: ref Selector, html: ref ValidHTML) -{IO}-> MountStatus; - -extern fn array_for_each(xs: ref [Element], f: fn(Element) -> ()) -{IO}-> (); -extern fn array_len(xs: ref [Element]) -> Int; - -// Example 1 — basic mount with success/error branches. -pub fn mount_app() -{IO}-> () { - mount_safe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - fn(el) -> () { Console::log("App mounted successfully"); }, - fn(err) -> () { Console::error("Mount failed: " ++ err); }, - ); -} - -// Example 2 — defer until DOM ready. -pub fn mount_when_dom_ready() -{IO}-> () { - mount_when_ready( - "#app", - "

App Title

", - fn(_el) -> () { Console::log("Mounted after DOM ready"); }, - fn(err) -> () { Console::error("Failed: " ++ err); }, - ); -} - -// Example 3 — atomic batch mount. -pub fn mount_multiple() -{IO}-> () { - let specs = [ - MountSpec #{ selector: "#header", html: "

Site Title

" }, - MountSpec #{ selector: "#nav", html: "" }, - MountSpec #{ selector: "#main", html: "

Content here

" }, - MountSpec #{ selector: "#footer", html: "
2026
" }, - ]; - - match mount_batch(specs) { - Mounted(elements) => { - Console::log("Batch mount succeeded"); - array_for_each(elements, fn(_el) -> () { Console::log(" element"); }); - }, - Failed(err) => { - Console::error("Batch mount failed (atomic — none mounted): " ++ err); - } - } -} - -// Example 4 — explicit two-stage validation before mounting. -pub fn mount_with_validation() -{IO}-> () { - match proven_selector_validate("#my-app") { - Err(e) => Console::error("Invalid selector: " ++ e), - Ok(valid_selector) => match proven_html_validate("
Content
") { - Err(e) => Console::error("Invalid HTML: " ++ e), - Ok(valid_html) => match mount(valid_selector, valid_html) { - Mounted(_el) => Console::log("Mounted with validated inputs"), - MountPointNotFound(s) => Console::error("Element not found: " ++ s), - InvalidSelector(_) => Console::error("impossible — already validated"), - InvalidHTML(_) => Console::error("impossible — already validated"), - }, - }, - } -} diff --git a/lithoglyph/analytics/examples/web-project-deno.json b/lithoglyph/analytics/examples/web-project-deno.json deleted file mode 100644 index 5ddd3bd7..00000000 --- a/lithoglyph/analytics/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "^12.0.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/lithoglyph/analytics/ffi/zig/build.zig b/lithoglyph/analytics/ffi/zig/build.zig deleted file mode 100644 index 4a2e049a..00000000 --- a/lithoglyph/analytics/ffi/zig/build.zig +++ /dev/null @@ -1,94 +0,0 @@ -// {{PROJECT}} FFI Build Configuration -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Shared library (.so, .dylib, .dll) - const lib = b.addSharedLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Set version - lib.version = .{ .major = 0, .minor = 1, .patch = 0 }; - - // Static library (.a) - const lib_static = b.addStaticLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Install artifacts - b.installArtifact(lib); - b.installArtifact(lib_static); - - // Generate header file for C compatibility - const header = b.addInstallHeader( - b.path("include/{{project}}.h"), - "{{project}}.h", - ); - b.getInstallStep().dependOn(&header.step); - - // Unit tests - const lib_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const run_lib_tests = b.addRunArtifact(lib_tests); - - const test_step = b.step("test", "Run library tests"); - test_step.dependOn(&run_lib_tests.step); - - // Integration tests - const integration_tests = b.addTest(.{ - .root_source_file = b.path("test/integration_test.zig"), - .target = target, - .optimize = optimize, - }); - - integration_tests.linkLibrary(lib); - - const run_integration_tests = b.addRunArtifact(integration_tests); - - const integration_test_step = b.step("test-integration", "Run integration tests"); - integration_test_step.dependOn(&run_integration_tests.step); - - // Documentation - const docs = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = .Debug, - }); - - const docs_step = b.step("docs", "Generate documentation"); - docs_step.dependOn(&b.addInstallDirectory(.{ - .source_dir = docs.getEmittedDocs(), - .install_dir = .prefix, - .install_subdir = "docs", - }).step); - - // Benchmark (if needed) - const bench = b.addExecutable(.{ - .name = "{{project}}-bench", - .root_source_file = b.path("bench/bench.zig"), - .target = target, - .optimize = .ReleaseFast, - }); - - bench.linkLibrary(lib); - - const run_bench = b.addRunArtifact(bench); - - const bench_step = b.step("bench", "Run benchmarks"); - bench_step.dependOn(&run_bench.step); -} diff --git a/lithoglyph/analytics/ffi/zig/src/main.zig b/lithoglyph/analytics/ffi/zig/src/main.zig deleted file mode 100644 index 6b233bc7..00000000 --- a/lithoglyph/analytics/ffi/zig/src/main.zig +++ /dev/null @@ -1,274 +0,0 @@ -// {{PROJECT}} FFI Implementation -// -// This module implements the C-compatible FFI declared in src/abi/Foreign.idr -// All types and layouts must match the Idris2 ABI definitions. -// -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -// Version information (keep in sync with project) -const VERSION = "0.1.0"; -const BUILD_INFO = "{{PROJECT}} built with Zig " ++ @import("builtin").zig_version_string; - -/// Thread-local error storage -threadlocal var last_error: ?[]const u8 = null; - -/// Set the last error message -fn setError(msg: []const u8) void { - last_error = msg; -} - -/// Clear the last error -fn clearError() void { - last_error = null; -} - -//============================================================================== -// Core Types (must match src/abi/Types.idr) -//============================================================================== - -/// Result codes (must match Idris2 Result type) -pub const Result = enum(c_int) { - ok = 0, - @"error" = 1, - invalid_param = 2, - out_of_memory = 3, - null_pointer = 4, -}; - -/// Library handle (opaque to prevent direct access) -pub const Handle = opaque { - // Internal state hidden from C - allocator: std.mem.Allocator, - initialized: bool, - // Add your fields here -}; - -//============================================================================== -// Library Lifecycle -//============================================================================== - -/// Initialize the library -/// Returns a handle, or null on failure -export fn {{project}}_init() ?*Handle { - const allocator = std.heap.c_allocator; - - const handle = allocator.create(Handle) catch { - setError("Failed to allocate handle"); - return null; - }; - - // Initialize handle - handle.* = .{ - .allocator = allocator, - .initialized = true, - }; - - clearError(); - return handle; -} - -/// Free the library handle -export fn {{project}}_free(handle: ?*Handle) void { - const h = handle orelse return; - const allocator = h.allocator; - - // Clean up resources - h.initialized = false; - - allocator.destroy(h); - clearError(); -} - -//============================================================================== -// Core Operations -//============================================================================== - -/// Process data (example operation) -export fn {{project}}_process(handle: ?*Handle, input: u32) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Example processing logic - _ = input; - - clearError(); - return .ok; -} - -//============================================================================== -// String Operations -//============================================================================== - -/// Get a string result (example) -/// Caller must free the returned string -export fn {{project}}_get_string(handle: ?*Handle) ?[*:0]const u8 { - const h = handle orelse { - setError("Null handle"); - return null; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return null; - } - - // Example: allocate and return a string - const result = h.allocator.dupeZ(u8, "Example result") catch { - setError("Failed to allocate string"); - return null; - }; - - clearError(); - return result.ptr; -} - -/// Free a string allocated by the library -export fn {{project}}_free_string(str: ?[*:0]const u8) void { - const s = str orelse return; - const allocator = std.heap.c_allocator; - - const slice = std.mem.span(s); - allocator.free(slice); -} - -//============================================================================== -// Array/Buffer Operations -//============================================================================== - -/// Process an array of data -export fn {{project}}_process_array( - handle: ?*Handle, - buffer: ?[*]const u8, - len: u32, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const buf = buffer orelse { - setError("Null buffer"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Access the buffer - const data = buf[0..len]; - _ = data; - - // Process data here - - clearError(); - return .ok; -} - -//============================================================================== -// Error Handling -//============================================================================== - -/// Get the last error message -/// Returns null if no error -export fn {{project}}_last_error() ?[*:0]const u8 { - const err = last_error orelse return null; - - // Return C string (static storage, no need to free) - const allocator = std.heap.c_allocator; - const c_str = allocator.dupeZ(u8, err) catch return null; - return c_str.ptr; -} - -//============================================================================== -// Version Information -//============================================================================== - -/// Get the library version -export fn {{project}}_version() [*:0]const u8 { - return VERSION.ptr; -} - -/// Get build information -export fn {{project}}_build_info() [*:0]const u8 { - return BUILD_INFO.ptr; -} - -//============================================================================== -// Callback Support -//============================================================================== - -/// Callback function type (C ABI) -pub const Callback = *const fn (u64, u32) callconv(.C) u32; - -/// Register a callback -export fn {{project}}_register_callback( - handle: ?*Handle, - callback: ?Callback, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const cb = callback orelse { - setError("Null callback"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Store callback for later use - _ = cb; - - clearError(); - return .ok; -} - -//============================================================================== -// Utility Functions -//============================================================================== - -/// Check if handle is initialized -export fn {{project}}_is_initialized(handle: ?*Handle) u32 { - const h = handle orelse return 0; - return if (h.initialized) 1 else 0; -} - -//============================================================================== -// Tests -//============================================================================== - -test "lifecycle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try std.testing.expect({{project}}_is_initialized(handle) == 1); -} - -test "error handling" { - const result = {{project}}_process(null, 0); - try std.testing.expectEqual(Result.null_pointer, result); - - const err = {{project}}_last_error(); - try std.testing.expect(err != null); -} - -test "version" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - try std.testing.expectEqualStrings(VERSION, ver_str); -} diff --git a/lithoglyph/analytics/ffi/zig/test/integration_test.zig b/lithoglyph/analytics/ffi/zig/test/integration_test.zig deleted file mode 100644 index 03419949..00000000 --- a/lithoglyph/analytics/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,182 +0,0 @@ -// {{PROJECT}} Integration Tests -// SPDX-License-Identifier: MPL-2.0 -// -// These tests verify that the Zig FFI correctly implements the Idris2 ABI - -const std = @import("std"); -const testing = std.testing; - -// Import FFI functions -extern fn {{project}}_init() ?*opaque {}; -extern fn {{project}}_free(?*opaque {}) void; -extern fn {{project}}_process(?*opaque {}, u32) c_int; -extern fn {{project}}_get_string(?*opaque {}) ?[*:0]const u8; -extern fn {{project}}_free_string(?[*:0]const u8) void; -extern fn {{project}}_last_error() ?[*:0]const u8; -extern fn {{project}}_version() [*:0]const u8; -extern fn {{project}}_is_initialized(?*opaque {}) u32; - -//============================================================================== -// Lifecycle Tests -//============================================================================== - -test "create and destroy handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try testing.expect(handle != null); -} - -test "handle is initialized" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const initialized = {{project}}_is_initialized(handle); - try testing.expectEqual(@as(u32, 1), initialized); -} - -test "null handle is not initialized" { - const initialized = {{project}}_is_initialized(null); - try testing.expectEqual(@as(u32, 0), initialized); -} - -//============================================================================== -// Operation Tests -//============================================================================== - -test "process with valid handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const result = {{project}}_process(handle, 42); - try testing.expectEqual(@as(c_int, 0), result); // 0 = ok -} - -test "process with null handle returns error" { - const result = {{project}}_process(null, 42); - try testing.expectEqual(@as(c_int, 4), result); // 4 = null_pointer -} - -//============================================================================== -// String Tests -//============================================================================== - -test "get string result" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const str = {{project}}_get_string(handle); - defer if (str) |s| {{project}}_free_string(s); - - try testing.expect(str != null); -} - -test "get string with null handle" { - const str = {{project}}_get_string(null); - try testing.expect(str == null); -} - -//============================================================================== -// Error Handling Tests -//============================================================================== - -test "last error after null handle operation" { - _ = {{project}}_process(null, 0); - - const err = {{project}}_last_error(); - try testing.expect(err != null); - - if (err) |e| { - const err_str = std.mem.span(e); - try testing.expect(err_str.len > 0); - } -} - -test "no error after successful operation" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - _ = {{project}}_process(handle, 0); - - // Error should be cleared after successful operation - // (This depends on implementation) -} - -//============================================================================== -// Version Tests -//============================================================================== - -test "version string is not empty" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - try testing.expect(ver_str.len > 0); -} - -test "version string is semantic version format" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - // Should be in format X.Y.Z - try testing.expect(std.mem.count(u8, ver_str, ".") >= 1); -} - -//============================================================================== -// Memory Safety Tests -//============================================================================== - -test "multiple handles are independent" { - const h1 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h1); - - const h2 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h2); - - try testing.expect(h1 != h2); - - // Operations on h1 should not affect h2 - _ = {{project}}_process(h1, 1); - _ = {{project}}_process(h2, 2); -} - -test "double free is safe" { - const handle = {{project}}_init() orelse return error.InitFailed; - - {{project}}_free(handle); - {{project}}_free(handle); // Should not crash -} - -test "free null is safe" { - {{project}}_free(null); // Should not crash -} - -//============================================================================== -// Thread Safety Tests (if applicable) -//============================================================================== - -test "concurrent operations" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const ThreadContext = struct { - h: *opaque {}, - id: u32, - }; - - const thread_fn = struct { - fn run(ctx: ThreadContext) void { - _ = {{project}}_process(ctx.h, ctx.id); - } - }.run; - - var threads: [4]std.Thread = undefined; - for (&threads, 0..) |*thread, i| { - thread.* = try std.Thread.spawn(.{}, thread_fn, .{ - ThreadContext{ .h = handle, .id = @intCast(i) }, - }); - } - - for (threads) |thread| { - thread.join(); - } -} diff --git a/lithoglyph/analytics/rescript.json b/lithoglyph/analytics/rescript.json deleted file mode 100644 index d1c78254..00000000 --- a/lithoglyph/analytics/rescript.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "@lith/analytics", - "sources": [{ "dir": "src" }], - "package-specs": [{ "module": "es6", "in-source": true }], - "suffix": ".res.js", - "bs-dependencies": ["@rescript/core"], - "bsc-flags": ["-open RescriptCore"] -} diff --git a/lithoglyph/analytics/src/LithAnalytics.jl b/lithoglyph/analytics/src/LithAnalytics.jl deleted file mode 100644 index 9efbe1e5..00000000 --- a/lithoglyph/analytics/src/LithAnalytics.jl +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -Lith-Analytics: OLAP analytics layer for Lith - -Provides columnar storage and analytical queries over Lith documents. -""" -module LithAnalytics - -using Arrow -using CSV -using Dates -using DataFrames -using HTTP -using JSON3 -using Oxygen -using Parquet2 -using Tables -using TOML -using UUIDs - -export Config, load_config -export LithClient, Document, fetch_collection, fetch_document, extract_prompt_scores, extract_timestamp, health_check -export ColumnarStore, sync!, load!, query, stats, prompt_stats, prompt_distribution, time_series, contributors -export serve - -include("config.jl") -include("lith_client.jl") -include("columnar_store.jl") -include("analytics.jl") -include("api.jl") - -end # module diff --git a/lithoglyph/analytics/src/Lith_Analytics_Aggregations.res b/lithoglyph/analytics/src/Lith_Analytics_Aggregations.res deleted file mode 100644 index 91057980..00000000 --- a/lithoglyph/analytics/src/Lith_Analytics_Aggregations.res +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Analytics Aggregations - * - * Statistical aggregation functions for data analysis - */ - -/** Aggregation type */ -type aggregationType = - | Count - | Sum - | Avg - | Min - | Max - | Median - | Stddev - | Variance - | Percentile(float) - | CountDistinct - -/** Aggregation result */ -type aggregationResult = { - aggregationType: aggregationType, - field: string, - value: float, - count: int, -} - -/** Group by result */ -type groupByResult = { - groupKey: Js.Dict.t, - aggregations: array, - count: int, -} - -/** Extract numeric value from JSON */ -let extractNumeric = (json: Js.Json.t): option => { - switch Js.Json.classify(json) { - | JSONNumber(n) => Some(n) - | JSONString(s) => Float.fromString(s) - | _ => None - } -} - -/** Count aggregation */ -let count = (values: array): float => { - Int.toFloat(Array.length(values)) -} - -/** Sum aggregation */ -let sum = (values: array): float => { - values->Array.reduce(0.0, (acc, v) => acc +. v) -} - -/** Average aggregation */ -let avg = (values: array): float => { - let len = Array.length(values) - if len == 0 { - 0.0 - } else { - sum(values) /. Int.toFloat(len) - } -} - -/** Minimum aggregation */ -let minValue = (values: array): option => { - if Array.length(values) == 0 { - None - } else { - Some(values->Array.reduce(Float.Constants.positiveInfinity, (acc, v) => min(acc, v))) - } -} - -/** Maximum aggregation */ -let maxValue = (values: array): option => { - if Array.length(values) == 0 { - None - } else { - Some(values->Array.reduce(Float.Constants.negativeInfinity, (acc, v) => max(acc, v))) - } -} - -/** Median aggregation */ -let median = (values: array): option => { - let len = Array.length(values) - if len == 0 { - None - } else { - let sorted = values->Array.toSorted((a, b) => a -. b) - if mod(len, 2) == 0 { - let mid = len / 2 - switch (sorted->Array.get(mid - 1), sorted->Array.get(mid)) { - | (Some(a), Some(b)) => Some((a +. b) /. 2.0) - | _ => None - } - } else { - sorted->Array.get(len / 2) - } - } -} - -/** Variance aggregation */ -let variance = (values: array): float => { - let len = Array.length(values) - if len == 0 { - 0.0 - } else { - let mean = avg(values) - let sumSquares = values->Array.reduce(0.0, (acc, v) => { - let diff = v -. mean - acc +. diff *. diff - }) - sumSquares /. Int.toFloat(len) - } -} - -/** Standard deviation aggregation */ -let stddev = (values: array): float => { - Js.Math.sqrt(variance(values)) -} - -/** Percentile aggregation */ -let percentile = (values: array, p: float): option => { - let len = Array.length(values) - if len == 0 || p < 0.0 || p > 100.0 { - None - } else { - let sorted = values->Array.toSorted((a, b) => a -. b) - let index = (p /. 100.0) *. Int.toFloat(len - 1) - let lower = Float.toInt(Js.Math.floor(index)) - let upper = Float.toInt(Js.Math.ceil(index)) - let fraction = index -. Js.Math.floor(index) - - switch (sorted->Array.get(lower), sorted->Array.get(upper)) { - | (Some(l), Some(u)) => Some(l +. fraction *. (u -. l)) - | (Some(l), None) => Some(l) - | _ => None - } - } -} - -/** Count distinct aggregation */ -let countDistinct = (values: array): int => { - let seen = Js.Dict.empty() - values->Array.forEach(v => { - let key = Js.Json.stringify(v) - Js.Dict.set(seen, key, true) - }) - Js.Dict.keys(seen)->Array.length -} - -/** Apply aggregation to field values */ -let applyAggregation = ( - aggregationType: aggregationType, - values: array, - field: string, -): aggregationResult => { - let numericValues = values->Array.filterMap(extractNumeric) - let count = Array.length(values) - - let value = switch aggregationType { - | Count => Int.toFloat(count) - | Sum => sum(numericValues) - | Avg => avg(numericValues) - | Min => minValue(numericValues)->Option.getOr(0.0) - | Max => maxValue(numericValues)->Option.getOr(0.0) - | Median => median(numericValues)->Option.getOr(0.0) - | Stddev => stddev(numericValues) - | Variance => variance(numericValues) - | Percentile(p) => percentile(numericValues, p)->Option.getOr(0.0) - | CountDistinct => Int.toFloat(countDistinct(values)) - } - - {aggregationType, field, value, count} -} - -/** Group documents by fields */ -let groupBy = ( - documents: array>, - groupFields: array, - aggregations: array<(aggregationType, string)>, -): array => { - // Build groups - let groups: Js.Dict.t>> = Js.Dict.empty() - - documents->Array.forEach(doc => { - // Build group key - let keyParts: array = [] - groupFields->Array.forEach(field => { - switch Js.Dict.get(doc, field) { - | Some(v) => keyParts->Array.push(Js.Json.stringify(v))->ignore - | None => keyParts->Array.push("null")->ignore - } - }) - let key = keyParts->Array.join("|") - - // Add to group - switch Js.Dict.get(groups, key) { - | Some(arr) => arr->Array.push(doc)->ignore - | None => Js.Dict.set(groups, key, [doc]) - } - }) - - // Process each group - Js.Dict.keys(groups)->Array.filterMap(key => { - switch Js.Dict.get(groups, key) { - | Some(docs) => { - // Build group key dict - let groupKey = Js.Dict.empty() - switch docs->Array.get(0) { - | Some(firstDoc) => - groupFields->Array.forEach(field => { - switch Js.Dict.get(firstDoc, field) { - | Some(v) => Js.Dict.set(groupKey, field, v) - | None => () - } - }) - | None => () - } - - // Apply aggregations - let aggResults = aggregations->Array.map(((aggType, field)) => { - let values = docs->Array.filterMap(doc => Js.Dict.get(doc, field)) - applyAggregation(aggType, values, field) - }) - - Some({ - groupKey, - aggregations: aggResults, - count: Array.length(docs), - }) - } - | None => None - } - }) -} - -/** Aggregation type to string */ -let aggregationTypeToString = (t: aggregationType): string => { - switch t { - | Count => "count" - | Sum => "sum" - | Avg => "avg" - | Min => "min" - | Max => "max" - | Median => "median" - | Stddev => "stddev" - | Variance => "variance" - | Percentile(p) => `percentile(${Float.toString(p)})` - | CountDistinct => "count_distinct" - } -} diff --git a/lithoglyph/analytics/src/Lith_Analytics_Export.res b/lithoglyph/analytics/src/Lith_Analytics_Export.res deleted file mode 100644 index a779a153..00000000 --- a/lithoglyph/analytics/src/Lith_Analytics_Export.res +++ /dev/null @@ -1,308 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Analytics Export - * - * Data export for visualization and reporting - */ - -/** Export format */ -type exportFormat = - | CSV - | JSON - | ChartJS - | D3 - | Vega - | TableData - -/** Chart type */ -type chartType = - | Line - | Bar - | Pie - | Scatter - | Area - | Histogram - | Heatmap - -/** Chart configuration */ -type chartConfig = { - chartType: chartType, - title: string, - xAxis: string, - yAxis: string, - series: array, - colors: option>, - legend: bool, -} - -/** Export options */ -type exportOptions = { - format: exportFormat, - chartConfig: option, - includeHeaders: bool, - dateFormat: string, - decimalPlaces: int, -} - -/** Default export options */ -let defaultOptions: exportOptions = { - format: JSON, - chartConfig: None, - includeHeaders: true, - dateFormat: "ISO", - decimalPlaces: 2, -} - -/** Format number with decimal places */ -let formatNumber = (n: float, decimals: int): string => { - let multiplier = Js.Math.pow_float(~base=10.0, ~exp=Int.toFloat(decimals)) - Float.toString(Js.Math.round(n *. multiplier) /. multiplier) -} - -/** Export to CSV */ -let toCSV = ( - data: array>, - options: exportOptions, -): string => { - if Array.length(data) == 0 { - "" - } else { - let lines: array = [] - - // Get headers from first row - let headers = switch data->Array.get(0) { - | Some(row) => Js.Dict.keys(row) - | None => [] - } - - // Add header line - if options.includeHeaders { - lines->Array.push(headers->Array.join(","))->ignore - } - - // Add data lines - data->Array.forEach(row => { - let values = headers->Array.map(h => { - switch Js.Dict.get(row, h) { - | Some(v) => - switch Js.Json.classify(v) { - | JSONString(s) => - if String.includes(s, ",") || String.includes(s, "\"") { - `"${String.replaceAll(s, "\"", "\"\"")}"` - } else { - s - } - | JSONNumber(n) => formatNumber(n, options.decimalPlaces) - | JSONTrue => "true" - | JSONFalse => "false" - | JSONNull => "" - | JSONObject(_) | JSONArray(_) => Js.Json.stringify(v) - } - | None => "" - } - }) - lines->Array.push(values->Array.join(","))->ignore - }) - - lines->Array.join("\n") - } -} - -/** Export to Chart.js format */ -let toChartJS = ( - data: array>, - config: chartConfig, -): Js.Json.t => { - // Extract labels (x-axis values) - let labels = data->Array.filterMap(row => - switch Js.Dict.get(row, config.xAxis) { - | Some(v) => Some(Js.Json.stringify(v)) - | None => None - } - ) - - // Extract datasets - let datasets = config.series->Array.mapWithIndex((series, i) => { - let values = data->Array.filterMap(row => - switch Js.Dict.get(row, series) { - | Some(v) => - switch Js.Json.classify(v) { - | JSONNumber(n) => Some(n) - | _ => None - } - | None => None - } - ) - - let color = switch config.colors { - | Some(colors) => colors->Array.get(mod(i, Array.length(colors)))->Option.getOr("#000000") - | None => "#000000" - } - - let obj = Js.Dict.empty() - Js.Dict.set(obj, "label", Js.Json.string(series)) - Js.Dict.set(obj, "data", Js.Json.array(values->Array.map(Js.Json.number))) - Js.Dict.set(obj, "borderColor", Js.Json.string(color)) - Js.Dict.set(obj, "backgroundColor", Js.Json.string(color)) - Js.Json.object_(obj) - }) - - let chartTypeStr = switch config.chartType { - | Line => "line" - | Bar => "bar" - | Pie => "pie" - | Scatter => "scatter" - | Area => "line" - | Histogram => "bar" - | Heatmap => "scatter" - } - - let result = Js.Dict.empty() - Js.Dict.set(result, "type", Js.Json.string(chartTypeStr)) - - let dataObj = Js.Dict.empty() - Js.Dict.set(dataObj, "labels", Js.Json.array(labels->Array.map(Js.Json.string))) - Js.Dict.set(dataObj, "datasets", Js.Json.array(datasets)) - Js.Dict.set(result, "data", Js.Json.object_(dataObj)) - - let optionsObj = Js.Dict.empty() - let pluginsObj = Js.Dict.empty() - let titleObj = Js.Dict.empty() - Js.Dict.set(titleObj, "display", Js.Json.boolean(true)) - Js.Dict.set(titleObj, "text", Js.Json.string(config.title)) - Js.Dict.set(pluginsObj, "title", Js.Json.object_(titleObj)) - let legendObj = Js.Dict.empty() - Js.Dict.set(legendObj, "display", Js.Json.boolean(config.legend)) - Js.Dict.set(pluginsObj, "legend", Js.Json.object_(legendObj)) - Js.Dict.set(optionsObj, "plugins", Js.Json.object_(pluginsObj)) - Js.Dict.set(result, "options", Js.Json.object_(optionsObj)) - - Js.Json.object_(result) -} - -/** Export to Vega-Lite format */ -let toVegaLite = ( - data: array>, - config: chartConfig, -): Js.Json.t => { - let markType = switch config.chartType { - | Line => "line" - | Bar => "bar" - | Pie => "arc" - | Scatter => "point" - | Area => "area" - | Histogram => "bar" - | Heatmap => "rect" - } - - let result = Js.Dict.empty() - Js.Dict.set(result, "$schema", Js.Json.string("https://vega.github.io/schema/vega-lite/v5.json")) - Js.Dict.set(result, "title", Js.Json.string(config.title)) - - // Data - let dataObj = Js.Dict.empty() - Js.Dict.set(dataObj, "values", Js.Json.array(data->Array.map(row => Js.Json.object_(row)))) - Js.Dict.set(result, "data", Js.Json.object_(dataObj)) - - // Mark - Js.Dict.set(result, "mark", Js.Json.string(markType)) - - // Encoding - let encodingObj = Js.Dict.empty() - - let xObj = Js.Dict.empty() - Js.Dict.set(xObj, "field", Js.Json.string(config.xAxis)) - Js.Dict.set(xObj, "type", Js.Json.string("nominal")) - Js.Dict.set(encodingObj, "x", Js.Json.object_(xObj)) - - let yObj = Js.Dict.empty() - Js.Dict.set(yObj, "field", Js.Json.string(config.yAxis)) - Js.Dict.set(yObj, "type", Js.Json.string("quantitative")) - Js.Dict.set(encodingObj, "y", Js.Json.object_(yObj)) - - Js.Dict.set(result, "encoding", Js.Json.object_(encodingObj)) - - Js.Json.object_(result) -} - -/** Export to D3 format (simple data structure) */ -let toD3 = (data: array>): Js.Json.t => { - Js.Json.array(data->Array.map(row => Js.Json.object_(row))) -} - -/** Export to table data format */ -let toTableData = (data: array>): Js.Json.t => { - if Array.length(data) == 0 { - let obj = Js.Dict.empty() - Js.Dict.set(obj, "columns", Js.Json.array([])) - Js.Dict.set(obj, "rows", Js.Json.array([])) - Js.Json.object_(obj) - } else { - let columns = switch data->Array.get(0) { - | Some(row) => Js.Dict.keys(row) - | None => [] - } - - let rows = data->Array.map(row => { - Js.Json.array( - columns->Array.map(col => - Js.Dict.get(row, col)->Option.getOr(Js.Json.null) - ), - ) - }) - - let obj = Js.Dict.empty() - Js.Dict.set(obj, "columns", Js.Json.array(columns->Array.map(Js.Json.string))) - Js.Dict.set(obj, "rows", Js.Json.array(rows)) - Js.Json.object_(obj) - } -} - -/** Main export function */ -let export = ( - data: array>, - options: exportOptions, -): string => { - switch options.format { - | CSV => toCSV(data, options) - | JSON => Js.Json.stringify(Js.Json.array(data->Array.map(row => Js.Json.object_(row)))) - | ChartJS => - switch options.chartConfig { - | Some(config) => Js.Json.stringify(toChartJS(data, config)) - | None => "{}" - } - | D3 => Js.Json.stringify(toD3(data)) - | Vega => - switch options.chartConfig { - | Some(config) => Js.Json.stringify(toVegaLite(data, config)) - | None => "{}" - } - | TableData => Js.Json.stringify(toTableData(data)) - } -} - -/** Chart type to string */ -let chartTypeToString = (t: chartType): string => { - switch t { - | Line => "line" - | Bar => "bar" - | Pie => "pie" - | Scatter => "scatter" - | Area => "area" - | Histogram => "histogram" - | Heatmap => "heatmap" - } -} - -/** Export format to string */ -let formatToString = (f: exportFormat): string => { - switch f { - | CSV => "csv" - | JSON => "json" - | ChartJS => "chartjs" - | D3 => "d3" - | Vega => "vega" - | TableData => "table" - } -} diff --git a/lithoglyph/analytics/src/Lith_Analytics_TimeSeries.res b/lithoglyph/analytics/src/Lith_Analytics_TimeSeries.res deleted file mode 100644 index 61ac2c71..00000000 --- a/lithoglyph/analytics/src/Lith_Analytics_TimeSeries.res +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Time Series Analytics - * - * Time-based data analysis and trend detection - */ - -/** Time bucket granularity */ -type timeGranularity = - | Second - | Minute - | Hour - | Day - | Week - | Month - | Quarter - | Year - -/** Time series data point */ -type dataPoint = { - timestamp: float, - value: float, - metadata: option>, -} - -/** Time bucket */ -type timeBucket = { - start: float, - end_: float, - points: array, - aggregatedValue: float, -} - -/** Time series */ -type timeSeries = { - name: string, - field: string, - granularity: timeGranularity, - buckets: array, - startTime: float, - endTime: float, -} - -/** Trend direction */ -type trendDirection = - | Increasing - | Decreasing - | Stable - | Volatile - -/** Trend analysis result */ -type trendAnalysis = { - direction: trendDirection, - slope: float, - rSquared: float, - changePercent: float, -} - -/** Granularity to milliseconds */ -let granularityToMs = (g: timeGranularity): float => { - switch g { - | Second => 1000.0 - | Minute => 60000.0 - | Hour => 3600000.0 - | Day => 86400000.0 - | Week => 604800000.0 - | Month => 2592000000.0 // Approximation (30 days) - | Quarter => 7776000000.0 // Approximation (90 days) - | Year => 31536000000.0 // Approximation (365 days) - } -} - -/** Get bucket start for timestamp */ -let getBucketStart = (timestamp: float, granularity: timeGranularity): float => { - let ms = granularityToMs(granularity) - Js.Math.floor(timestamp /. ms) *. ms -} - -/** Create time series from data points */ -let createTimeSeries = ( - name: string, - field: string, - points: array, - granularity: timeGranularity, -): timeSeries => { - if Array.length(points) == 0 { - { - name, - field, - granularity, - buckets: [], - startTime: 0.0, - endTime: 0.0, - } - } else { - // Sort by timestamp - let sorted = points->Array.toSorted((a, b) => a.timestamp -. b.timestamp) - - // Find time range - let startTime = switch sorted->Array.get(0) { - | Some(p) => getBucketStart(p.timestamp, granularity) - | None => 0.0 - } - let endTime = switch sorted->Array.get(Array.length(sorted) - 1) { - | Some(p) => getBucketStart(p.timestamp, granularity) +. granularityToMs(granularity) - | None => 0.0 - } - - // Group into buckets - let bucketMap: Js.Dict.t> = Js.Dict.empty() - - sorted->Array.forEach(point => { - let bucketStart = getBucketStart(point.timestamp, granularity) - let key = Float.toString(bucketStart) - switch Js.Dict.get(bucketMap, key) { - | Some(arr) => arr->Array.push(point)->ignore - | None => Js.Dict.set(bucketMap, key, [point]) - } - }) - - // Create buckets - let ms = granularityToMs(granularity) - let buckets: array = [] - let numBuckets = Float.toInt((endTime -. startTime) /. ms) - - for i in 0 to numBuckets - 1 { - let start = startTime +. Int.toFloat(i) *. ms - let key = Float.toString(start) - let points = Js.Dict.get(bucketMap, key)->Option.getOr([]) - let values = points->Array.map(p => p.value) - let aggregatedValue = if Array.length(values) == 0 { - 0.0 - } else { - values->Array.reduce(0.0, (a, b) => a +. b) /. Int.toFloat(Array.length(values)) - } - - buckets->Array.push({ - start, - end_: start +. ms, - points, - aggregatedValue, - })->ignore - } - - {name, field, granularity, buckets, startTime, endTime} - } -} - -/** Calculate linear regression */ -let linearRegression = (points: array<(float, float)>): (float, float, float) => { - let n = Int.toFloat(Array.length(points)) - if n < 2.0 { - (0.0, 0.0, 0.0) - } else { - let sumX = points->Array.reduce(0.0, (acc, (x, _)) => acc +. x) - let sumY = points->Array.reduce(0.0, (acc, (_, y)) => acc +. y) - let sumXY = points->Array.reduce(0.0, (acc, (x, y)) => acc +. x *. y) - let sumX2 = points->Array.reduce(0.0, (acc, (x, _)) => acc +. x *. x) - let sumY2 = points->Array.reduce(0.0, (acc, (_, y)) => acc +. y *. y) - - let slope = (n *. sumXY -. sumX *. sumY) /. (n *. sumX2 -. sumX *. sumX) - let intercept = (sumY -. slope *. sumX) /. n - - // R-squared - let ssTotal = sumY2 -. sumY *. sumY /. n - let ssResidual = points->Array.reduce(0.0, (acc, (x, y)) => { - let predicted = slope *. x +. intercept - let residual = y -. predicted - acc +. residual *. residual - }) - let rSquared = if ssTotal == 0.0 { - 1.0 - } else { - 1.0 -. ssResidual /. ssTotal - } - - (slope, intercept, rSquared) - } -} - -/** Analyze trend in time series */ -let analyzeTrend = (series: timeSeries): trendAnalysis => { - if Array.length(series.buckets) < 2 { - {direction: Stable, slope: 0.0, rSquared: 0.0, changePercent: 0.0} - } else { - // Create points for regression (index, value) - let points = series.buckets->Array.mapWithIndex((bucket, i) => - (Int.toFloat(i), bucket.aggregatedValue) - ) - - let (slope, _, rSquared) = linearRegression(points) - - // Calculate change percent - let firstValue = switch series.buckets->Array.get(0) { - | Some(b) => b.aggregatedValue - | None => 0.0 - } - let lastValue = switch series.buckets->Array.get(Array.length(series.buckets) - 1) { - | Some(b) => b.aggregatedValue - | None => 0.0 - } - let changePercent = if firstValue == 0.0 { - 0.0 - } else { - (lastValue -. firstValue) /. firstValue *. 100.0 - } - - // Determine direction - let direction = if rSquared < 0.3 { - Volatile - } else if slope > 0.01 { - Increasing - } else if slope < -0.01 { - Decreasing - } else { - Stable - } - - {direction, slope, rSquared, changePercent} - } -} - -/** Moving average */ -let movingAverage = (series: timeSeries, windowSize: int): array => { - let values = series.buckets->Array.map(b => b.aggregatedValue) - let len = Array.length(values) - - if len < windowSize { - values - } else { - let result: array = [] - for i in 0 to len - windowSize { - let sum = ref(0.0) - for j in 0 to windowSize - 1 { - switch values->Array.get(i + j) { - | Some(v) => sum := sum.contents +. v - | None => () - } - } - result->Array.push(sum.contents /. Int.toFloat(windowSize))->ignore - } - result - } -} - -/** Detect anomalies using standard deviation */ -let detectAnomalies = (series: timeSeries, threshold: float): array => { - let values = series.buckets->Array.map(b => b.aggregatedValue) - let len = Array.length(values) - - if len < 2 { - [] - } else { - let mean = values->Array.reduce(0.0, (a, b) => a +. b) /. Int.toFloat(len) - let variance = values->Array.reduce(0.0, (acc, v) => { - let diff = v -. mean - acc +. diff *. diff - }) /. Int.toFloat(len) - let stddev = Js.Math.sqrt(variance) - - series.buckets->Array.filter(bucket => { - let deviation = Js.Math.abs_float(bucket.aggregatedValue -. mean) - deviation > threshold *. stddev - }) - } -} - -/** Granularity to string */ -let granularityToString = (g: timeGranularity): string => { - switch g { - | Second => "second" - | Minute => "minute" - | Hour => "hour" - | Day => "day" - | Week => "week" - | Month => "month" - | Quarter => "quarter" - | Year => "year" - } -} - -/** Trend direction to string */ -let trendDirectionToString = (d: trendDirection): string => { - switch d { - | Increasing => "increasing" - | Decreasing => "decreasing" - | Stable => "stable" - | Volatile => "volatile" - } -} diff --git a/lithoglyph/analytics/src/Lith_Analytics_Window.res b/lithoglyph/analytics/src/Lith_Analytics_Window.res deleted file mode 100644 index 3616d7ce..00000000 --- a/lithoglyph/analytics/src/Lith_Analytics_Window.res +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Window Functions - * - * SQL-style window functions for advanced analytics - */ - -/** Window frame type */ -type frameType = - | Rows - | Range - -/** Frame boundary */ -type frameBoundary = - | UnboundedPreceding - | CurrentRow - | Preceding(int) - | Following(int) - | UnboundedFollowing - -/** Window specification */ -type windowSpec = { - partitionBy: array, - orderBy: array<(string, bool)>, // (field, ascending) - frameType: frameType, - frameStart: frameBoundary, - frameEnd: frameBoundary, -} - -/** Default window spec */ -let defaultWindowSpec: windowSpec = { - partitionBy: [], - orderBy: [], - frameType: Rows, - frameStart: UnboundedPreceding, - frameEnd: CurrentRow, -} - -/** Window function type */ -type windowFunction = - | RowNumber - | Rank - | DenseRank - | Ntile(int) - | Lag(string, int, option) - | Lead(string, int, option) - | FirstValue(string) - | LastValue(string) - | NthValue(string, int) - | CumulativeSum(string) - | RunningAvg(string) - | PercentRank - -/** Window result */ -type windowResult = { - rowIndex: int, - partitionKey: string, - value: Js.Json.t, -} - -/** Get partition key for document */ -let getPartitionKey = (doc: Js.Dict.t, fields: array): string => { - fields - ->Array.map(f => { - switch Js.Dict.get(doc, f) { - | Some(v) => Js.Json.stringify(v) - | None => "null" - } - }) - ->Array.join("|") -} - -/** Compare documents for ordering */ -let compareForOrder = ( - a: Js.Dict.t, - b: Js.Dict.t, - orderBy: array<(string, bool)>, -): int => { - let result = ref(0) - - orderBy->Array.forEach(((field, ascending)) => { - if result.contents == 0 { - let aVal = Js.Dict.get(a, field) - let bVal = Js.Dict.get(b, field) - - let cmp = switch (aVal, bVal) { - | (Some(av), Some(bv)) => { - let aStr = Js.Json.stringify(av) - let bStr = Js.Json.stringify(bv) - if aStr < bStr { - -1 - } else if aStr > bStr { - 1 - } else { - 0 - } - } - | (Some(_), None) => 1 - | (None, Some(_)) => -1 - | (None, None) => 0 - } - - result := if ascending { - cmp - } else { - -cmp - } - } - }) - - result.contents -} - -/** Get frame boundaries for a row */ -let getFrameRange = ( - currentIndex: int, - partitionSize: int, - frameStart: frameBoundary, - frameEnd: frameBoundary, -): (int, int) => { - let start = switch frameStart { - | UnboundedPreceding => 0 - | CurrentRow => currentIndex - | Preceding(n) => max(0, currentIndex - n) - | Following(n) => min(partitionSize - 1, currentIndex + n) - | UnboundedFollowing => partitionSize - 1 - } - - let end_ = switch frameEnd { - | UnboundedPreceding => 0 - | CurrentRow => currentIndex - | Preceding(n) => max(0, currentIndex - n) - | Following(n) => min(partitionSize - 1, currentIndex + n) - | UnboundedFollowing => partitionSize - 1 - } - - (start, end_) -} - -/** Apply window function to partition */ -let applyWindowFunction = ( - func: windowFunction, - partition: array>, - currentIndex: int, - spec: windowSpec, -): Js.Json.t => { - let (frameStart, frameEnd) = getFrameRange( - currentIndex, - Array.length(partition), - spec.frameStart, - spec.frameEnd, - ) - - switch func { - | RowNumber => Js.Json.number(Int.toFloat(currentIndex + 1)) - - | Rank => { - // Count how many rows have smaller values - let current = partition->Array.getUnsafe(currentIndex) - let rank = ref(1) - for i in 0 to currentIndex - 1 { - if compareForOrder(partition->Array.getUnsafe(i), current, spec.orderBy) < 0 { - rank := rank.contents + 1 - } - } - Js.Json.number(Int.toFloat(rank.contents)) - } - - | DenseRank => { - // Count distinct values before current - let current = partition->Array.getUnsafe(currentIndex) - let seen = Js.Dict.empty() - for i in 0 to currentIndex - 1 { - let key = spec.orderBy - ->Array.map(((f, _)) => { - switch Js.Dict.get(partition->Array.getUnsafe(i), f) { - | Some(v) => Js.Json.stringify(v) - | None => "null" - } - }) - ->Array.join("|") - Js.Dict.set(seen, key, true) - } - Js.Json.number(Int.toFloat(Js.Dict.keys(seen)->Array.length + 1)) - } - - | Ntile(n) => { - let partitionSize = Array.length(partition) - let bucket = Float.toInt(Int.toFloat(currentIndex * n) /. Int.toFloat(partitionSize)) + 1 - Js.Json.number(Int.toFloat(bucket)) - } - - | Lag(field, offset, default) => { - let targetIndex = currentIndex - offset - if targetIndex < 0 { - default->Option.getOr(Js.Json.null) - } else { - switch partition->Array.get(targetIndex) { - | Some(row) => Js.Dict.get(row, field)->Option.getOr(Js.Json.null) - | None => default->Option.getOr(Js.Json.null) - } - } - } - - | Lead(field, offset, default) => { - let targetIndex = currentIndex + offset - switch partition->Array.get(targetIndex) { - | Some(row) => Js.Dict.get(row, field)->Option.getOr(Js.Json.null) - | None => default->Option.getOr(Js.Json.null) - } - } - - | FirstValue(field) => { - switch partition->Array.get(frameStart) { - | Some(row) => Js.Dict.get(row, field)->Option.getOr(Js.Json.null) - | None => Js.Json.null - } - } - - | LastValue(field) => { - switch partition->Array.get(frameEnd) { - | Some(row) => Js.Dict.get(row, field)->Option.getOr(Js.Json.null) - | None => Js.Json.null - } - } - - | NthValue(field, n) => { - let targetIndex = frameStart + n - 1 - if targetIndex <= frameEnd { - switch partition->Array.get(targetIndex) { - | Some(row) => Js.Dict.get(row, field)->Option.getOr(Js.Json.null) - | None => Js.Json.null - } - } else { - Js.Json.null - } - } - - | CumulativeSum(field) => { - let sum = ref(0.0) - for i in frameStart to min(currentIndex, frameEnd) { - switch partition->Array.get(i) { - | Some(row) => - switch Js.Dict.get(row, field) { - | Some(v) => - switch Js.Json.classify(v) { - | JSONNumber(n) => sum := sum.contents +. n - | _ => () - } - | None => () - } - | None => () - } - } - Js.Json.number(sum.contents) - } - - | RunningAvg(field) => { - let sum = ref(0.0) - let count = ref(0) - for i in frameStart to min(currentIndex, frameEnd) { - switch partition->Array.get(i) { - | Some(row) => - switch Js.Dict.get(row, field) { - | Some(v) => - switch Js.Json.classify(v) { - | JSONNumber(n) => { - sum := sum.contents +. n - count := count.contents + 1 - } - | _ => () - } - | None => () - } - | None => () - } - } - if count.contents == 0 { - Js.Json.number(0.0) - } else { - Js.Json.number(sum.contents /. Int.toFloat(count.contents)) - } - } - - | PercentRank => { - if Array.length(partition) == 1 { - Js.Json.number(0.0) - } else { - let current = partition->Array.getUnsafe(currentIndex) - let rank = ref(0) - for i in 0 to currentIndex - 1 { - if compareForOrder(partition->Array.getUnsafe(i), current, spec.orderBy) < 0 { - rank := rank.contents + 1 - } - } - Js.Json.number(Int.toFloat(rank.contents) /. Int.toFloat(Array.length(partition) - 1)) - } - } - } -} - -/** Execute window function on documents */ -let execute = ( - documents: array>, - func: windowFunction, - spec: windowSpec, -): array => { - // Group by partition - let partitions: Js.Dict.t>> = Js.Dict.empty() - - documents->Array.forEach(doc => { - let key = getPartitionKey(doc, spec.partitionBy) - switch Js.Dict.get(partitions, key) { - | Some(arr) => arr->Array.push(doc)->ignore - | None => Js.Dict.set(partitions, key, [doc]) - } - }) - - // Process each partition - let results: array = [] - let globalIndex = ref(0) - - Js.Dict.keys(partitions)->Array.forEach(partitionKey => { - switch Js.Dict.get(partitions, partitionKey) { - | Some(partition) => { - // Sort partition by orderBy - let sorted = partition->Array.toSorted((a, b) => Float.toInt( - Int.toFloat(compareForOrder(a, b, spec.orderBy)), - )) - - // Apply window function to each row - sorted->Array.forEachWithIndex((_, i) => { - let value = applyWindowFunction(func, sorted, i, spec) - results - ->Array.push({ - rowIndex: globalIndex.contents, - partitionKey, - value, - }) - ->ignore - globalIndex := globalIndex.contents + 1 - }) - } - | None => () - } - }) - - results -} - -/** Window function to string */ -let windowFunctionToString = (f: windowFunction): string => { - switch f { - | RowNumber => "row_number()" - | Rank => "rank()" - | DenseRank => "dense_rank()" - | Ntile(n) => `ntile(${Int.toString(n)})` - | Lag(field, offset, _) => `lag(${field}, ${Int.toString(offset)})` - | Lead(field, offset, _) => `lead(${field}, ${Int.toString(offset)})` - | FirstValue(field) => `first_value(${field})` - | LastValue(field) => `last_value(${field})` - | NthValue(field, n) => `nth_value(${field}, ${Int.toString(n)})` - | CumulativeSum(field) => `sum(${field})` - | RunningAvg(field) => `avg(${field})` - | PercentRank => "percent_rank()" - } -} diff --git a/lithoglyph/analytics/src/analytics.jl b/lithoglyph/analytics/src/analytics.jl deleted file mode 100644 index 9377afde..00000000 --- a/lithoglyph/analytics/src/analytics.jl +++ /dev/null @@ -1,221 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -Analytics functions for PROMPT scores and time-series analysis -""" - -""" - prompt_stats(store::ColumnarStore, collection::String; groupby::Union{Symbol,Nothing}=nothing) - -Calculate PROMPT score statistics. -""" -function prompt_stats(store::ColumnarStore, collection::String; groupby::Union{Symbol,Nothing}=nothing) - df = query(store, collection) - - if isempty(df) - return Dict("error" => "Collection not found or empty") - end - - prompt_cols = filter(n -> startswith(string(n), "prompt_"), names(df)) - - if isempty(prompt_cols) - return Dict("error" => "No PROMPT scores found in collection") - end - - if isnothing(groupby) - # Overall statistics - result = Dict{String,Any}() - - for col in prompt_cols - dimension = replace(string(col), "prompt_" => "") - values = skipmissing(df[!, col]) - - if !isempty(collect(values)) - result[dimension] = Dict( - "mean" => mean(values), - "median" => median(collect(values)), - "min" => minimum(values), - "max" => maximum(values), - "count" => length(collect(values)) - ) - end - end - - return result - else - # Grouped statistics - group_col = Symbol("data_$groupby") - - if group_col ∉ Symbol.(names(df)) - return Dict("error" => "Group column not found: $groupby") - end - - grouped = groupby(df, group_col) - - result = Dict{String,Any}() - - for group in grouped - key = string(first(group[!, group_col])) - result[key] = Dict{String,Any}() - - for col in prompt_cols - dimension = replace(string(col), "prompt_" => "") - values = skipmissing(group[!, col]) - - if !isempty(collect(values)) - result[key][dimension] = Dict( - "mean" => mean(values), - "count" => length(collect(values)) - ) - end - end - end - - return result - end -end - -""" - prompt_distribution(store::ColumnarStore, collection::String, dimension::Symbol; bins::Int=10) - -Get distribution of a PROMPT dimension. -""" -function prompt_distribution(store::ColumnarStore, collection::String, dimension::Symbol; bins::Int=10) - df = query(store, collection) - - if isempty(df) - return Dict("error" => "Collection not found or empty") - end - - col = Symbol("prompt_$dimension") - - if col ∉ Symbol.(names(df)) - return Dict("error" => "PROMPT dimension not found: $dimension") - end - - values = collect(skipmissing(df[!, col])) - - if isempty(values) - return Dict("error" => "No values for dimension: $dimension") - end - - # Calculate histogram - min_val = minimum(values) - max_val = maximum(values) - bin_width = (max_val - min_val) / bins - - histogram = zeros(Int, bins) - for v in values - bin_idx = min(bins, max(1, ceil(Int, (v - min_val) / bin_width))) - histogram[bin_idx] += 1 - end - - return Dict( - "dimension" => string(dimension), - "min" => min_val, - "max" => max_val, - "bin_width" => bin_width, - "histogram" => histogram, - "total" => length(values) - ) -end - -""" - time_series(store::ColumnarStore, collection::String; interval::Symbol=:day, field::Union{Symbol,Nothing}=nothing) - -Generate time-series data for document counts or field values. -""" -function time_series(store::ColumnarStore, collection::String; interval::Symbol=:day, field::Union{Symbol,Nothing}=nothing) - df = query(store, collection) - - if isempty(df) - return Dict("error" => "Collection not found or empty") - end - - if :timestamp ∉ Symbol.(names(df)) - return Dict("error" => "No timestamp column in collection") - end - - # Filter rows with valid timestamps - df_with_ts = filter(row -> !ismissing(row.timestamp), df) - - if isempty(df_with_ts) - return Dict("error" => "No rows with valid timestamps") - end - - # Truncate timestamps to interval - truncate_fn = if interval == :hour - t -> DateTime(year(t), month(t), day(t), hour(t)) - elseif interval == :day - t -> DateTime(year(t), month(t), day(t)) - elseif interval == :week - t -> DateTime(year(t), month(t), day(t) - dayofweek(t) + 1) - elseif interval == :month - t -> DateTime(year(t), month(t)) - else - error("Invalid interval: $interval") - end - - df_with_ts[!, :period] = truncate_fn.(df_with_ts.timestamp) - - if isnothing(field) - # Count documents per period - grouped = DataFrames.groupby(df_with_ts, :period) - result_df = combine(grouped, nrow => :count) - sort!(result_df, :period) - - return Dict( - "interval" => string(interval), - "data" => [Dict("period" => string(row.period), "count" => row.count) for row in eachrow(result_df)] - ) - else - # Aggregate field per period - field_col = Symbol("data_$field") - - if field_col ∉ Symbol.(names(df_with_ts)) - return Dict("error" => "Field not found: $field") - end - - grouped = DataFrames.groupby(df_with_ts, :period) - result_df = combine(grouped, field_col => mean => :mean_value, nrow => :count) - sort!(result_df, :period) - - return Dict( - "interval" => string(interval), - "field" => string(field), - "data" => [Dict("period" => string(row.period), "mean" => row.mean_value, "count" => row.count) for row in eachrow(result_df)] - ) - end -end - -""" - contributors(store::ColumnarStore, collection::String) - -Analyze contributors/provenance data. -""" -function contributors(store::ColumnarStore, collection::String) - df = query(store, collection) - - if isempty(df) - return Dict("error" => "Collection not found or empty") - end - - if :created_by ∉ Symbol.(names(df)) - return Dict("error" => "No provenance data in collection") - end - - # Filter rows with created_by - df_with_prov = filter(row -> !ismissing(row.created_by), df) - - if isempty(df_with_prov) - return Dict("error" => "No provenance data found") - end - - grouped = DataFrames.groupby(df_with_prov, :created_by) - result_df = combine(grouped, nrow => :document_count) - sort!(result_df, :document_count, rev=true) - - return Dict( - "total_contributors" => nrow(result_df), - "contributors" => [Dict("name" => row.created_by, "documents" => row.document_count) for row in eachrow(result_df)] - ) -end diff --git a/lithoglyph/analytics/src/api.jl b/lithoglyph/analytics/src/api.jl deleted file mode 100644 index 7a2fbdb7..00000000 --- a/lithoglyph/analytics/src/api.jl +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -HTTP API for Lith-Analytics using Oxygen.jl -""" - -""" -Application state shared across handlers -""" -mutable struct AppState - client::LithClient - store::ColumnarStore - config::Config -end - -# Global state (Oxygen uses global handlers) -const APP_STATE = Ref{Union{AppState,Nothing}}(nothing) - -""" - serve(config::Config) - -Start the HTTP server. -""" -function serve(config::Config) - # Initialize state - client = LithClient(config.lith.api_url) - store = ColumnarStore(config.storage.data_dir) - - # Load existing data - for collection in config.lith.collections - load!(store, collection) - end - - APP_STATE[] = AppState(client, store, config) - - @info "Lith-Analytics starting" host=config.server.host port=config.server.port - - # Define routes - @get "/analytics/health" function() - state = APP_STATE[] - lith_reachable = health_check(state.client) - - return Dict( - "status" => "ok", - "lith_reachable" => lith_reachable, - "collections" => length(state.store.collections) - ) - end - - @get "/analytics/stats" function() - state = APP_STATE[] - return stats(state.store) - end - - @get "/analytics/collections" function() - state = APP_STATE[] - result = Dict{String,Any}() - - for (collection, df) in state.store.collections - result[collection] = Dict( - "rows" => nrow(df), - "columns" => names(df), - "last_sync" => get(state.store.last_sync, collection, nothing) - ) - end - - return result - end - - @post "/analytics/sync" function(req) - state = APP_STATE[] - body = JSON3.read(String(req.body)) - - collection = get(body, :collection, nothing) - mode = Symbol(get(body, :mode, "full")) - - if isnothing(collection) - return Dict("error" => "Collection required") - end - - start_time = now() - - try - rows = sync!(state.store, state.client, string(collection); mode=mode) - duration = now() - start_time - - return Dict( - "status" => "ok", - "collection" => collection, - "rows_synced" => rows, - "duration_ms" => Dates.value(duration) - ) - catch e - return Dict("error" => string(e)) - end - end - - @get "/analytics/prompt-scores" function(req) - state = APP_STATE[] - params = Oxygen.queryparams(req) - - collection = get(params, "collection", "evidence") - groupby = get(params, "groupBy", nothing) - - return prompt_stats(state.store, collection; groupby=isnothing(groupby) ? nothing : Symbol(groupby)) - end - - @get "/analytics/prompt-distribution" function(req) - state = APP_STATE[] - params = Oxygen.queryparams(req) - - collection = get(params, "collection", "evidence") - dimension = get(params, "dimension", "provenance") - bins = parse(Int, get(params, "bins", "10")) - - return prompt_distribution(state.store, collection, Symbol(dimension); bins=bins) - end - - @get "/analytics/time-series" function(req) - state = APP_STATE[] - params = Oxygen.queryparams(req) - - collection = get(params, "collection", "evidence") - interval = Symbol(get(params, "interval", "day")) - field = get(params, "field", nothing) - - return time_series(state.store, collection; interval=interval, field=isnothing(field) ? nothing : Symbol(field)) - end - - @get "/analytics/contributors" function(req) - state = APP_STATE[] - params = Oxygen.queryparams(req) - - collection = get(params, "collection", "evidence") - - return contributors(state.store, collection) - end - - # Start server - Oxygen.serve(; host=config.server.host, port=config.server.port) -end diff --git a/lithoglyph/analytics/src/columnar_store.jl b/lithoglyph/analytics/src/columnar_store.jl deleted file mode 100644 index b262eadb..00000000 --- a/lithoglyph/analytics/src/columnar_store.jl +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -Columnar storage using Arrow/Parquet for analytical queries -""" - -""" -Columnar store for analytical data -""" -mutable struct ColumnarStore - data_dir::String - collections::Dict{String,DataFrame} - last_sync::Dict{String,DateTime} - - function ColumnarStore(data_dir::String) - mkpath(data_dir) - new(data_dir, Dict{String,DataFrame}(), Dict{String,DateTime}()) - end -end - -""" - sync!(store::ColumnarStore, client::LithClient, collection::String; mode::Symbol=:full) - -Sync data from Lith to columnar store. -Modes: :full (replace all), :incremental (append new) -""" -function sync!(store::ColumnarStore, client::LithClient, collection::String; mode::Symbol=:full) - start_time = now() - - @info "Starting sync" collection mode - - documents = fetch_collection(client, collection) - - # Transform to columnar format - rows = [] - for doc in documents - row = Dict{String,Any}( - "id" => doc.id, - "synced_at" => now() - ) - - # Flatten document data (first level only) - for (key, value) in doc.data - if value isa Number || value isa String || value isa Bool - row["data_$key"] = value - elseif value isa Dict - # Handle nested objects like PROMPT scores - for (k2, v2) in value - if v2 isa Number || v2 isa String || v2 isa Bool - row["data_$(key)_$k2"] = v2 - end - end - end - end - - # Extract PROMPT scores - prompt = extract_prompt_scores(doc) - if !isnothing(prompt) - row["prompt_provenance"] = coalesce(prompt.provenance, missing) - row["prompt_replicability"] = coalesce(prompt.replicability, missing) - row["prompt_objective"] = coalesce(prompt.objective, missing) - row["prompt_methodology"] = coalesce(prompt.methodology, missing) - row["prompt_publication"] = coalesce(prompt.publication, missing) - row["prompt_transparency"] = coalesce(prompt.transparency, missing) - end - - # Extract timestamp - ts = extract_timestamp(doc) - row["timestamp"] = isnothing(ts) ? missing : ts - - # Provenance - if !isnothing(doc.provenance) - row["created_by"] = get(doc.provenance, "created_by", missing) - end - - push!(rows, row) - end - - # Create DataFrame - if !isempty(rows) - df = DataFrame(rows) - - if mode == :full - store.collections[collection] = df - else - # Incremental: append new rows - existing = get(store.collections, collection, DataFrame()) - if isempty(existing) - store.collections[collection] = df - else - store.collections[collection] = vcat(existing, df) - end - end - - # Persist to Parquet - parquet_path = joinpath(store.data_dir, "$(collection).parquet") - Parquet2.writefile(parquet_path, store.collections[collection]) - - store.last_sync[collection] = now() - - duration = now() - start_time - @info "Sync complete" collection rows=nrow(store.collections[collection]) duration - end - - return nrow(get(store.collections, collection, DataFrame())) -end - -""" - load!(store::ColumnarStore, collection::String) - -Load a collection from Parquet file if it exists. -""" -function load!(store::ColumnarStore, collection::String) - parquet_path = joinpath(store.data_dir, "$(collection).parquet") - - if isfile(parquet_path) - @info "Loading from Parquet" collection path=parquet_path - store.collections[collection] = DataFrame(Parquet2.readfile(parquet_path)) - @info "Loaded" collection rows=nrow(store.collections[collection]) - end -end - -""" - query(store::ColumnarStore, collection::String) -> DataFrame - -Get the DataFrame for a collection. -""" -function query(store::ColumnarStore, collection::String)::DataFrame - return get(store.collections, collection, DataFrame()) -end - -""" - stats(store::ColumnarStore) -> Dict - -Get statistics about the store. -""" -function stats(store::ColumnarStore) - result = Dict{String,Any}() - - for (collection, df) in store.collections - result[collection] = Dict( - "rows" => nrow(df), - "columns" => ncol(df), - "last_sync" => get(store.last_sync, collection, nothing) - ) - end - - return result -end diff --git a/lithoglyph/analytics/src/config.jl b/lithoglyph/analytics/src/config.jl deleted file mode 100644 index 53a1482b..00000000 --- a/lithoglyph/analytics/src/config.jl +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -Configuration management for Lith-Analytics -""" - -""" -Lith connection configuration -""" -Base.@kwdef struct LithConfig - api_url::String = "http://localhost:8080" - collections::Vector{String} = ["evidence"] -end - -""" -HTTP server configuration -""" -Base.@kwdef struct ServerConfig - host::String = "127.0.0.1" - port::Int = 8082 -end - -""" -Storage configuration -""" -Base.@kwdef struct StorageConfig - data_dir::String = "./data" - retention_days::Int = 0 -end - -""" -Sync configuration -""" -Base.@kwdef struct SyncConfig - auto_sync_minutes::Int = 0 -end - -""" -Main configuration structure -""" -Base.@kwdef struct Config - lith::LithConfig = LithConfig() - server::ServerConfig = ServerConfig() - storage::StorageConfig = StorageConfig() - sync::SyncConfig = SyncConfig() -end - -""" - load_config(path::String) -> Config - -Load configuration from a TOML file. Returns default config if file doesn't exist. -""" -function load_config(path::String)::Config - if !isfile(path) - @warn "Config file not found, using defaults" path - return Config() - end - - data = TOML.parsefile(path) - - lith = if haskey(data, "lith") - f = data["lith"] - LithConfig( - api_url = get(f, "api_url", "http://localhost:8080"), - collections = get(f, "collections", ["evidence"]) - ) - else - LithConfig() - end - - server = if haskey(data, "server") - s = data["server"] - ServerConfig( - host = get(s, "host", "127.0.0.1"), - port = get(s, "port", 8082) - ) - else - ServerConfig() - end - - storage = if haskey(data, "storage") - st = data["storage"] - StorageConfig( - data_dir = get(st, "data_dir", "./data"), - retention_days = get(st, "retention_days", 0) - ) - else - StorageConfig() - end - - sync = if haskey(data, "sync") - sy = data["sync"] - SyncConfig( - auto_sync_minutes = get(sy, "auto_sync_minutes", 0) - ) - else - SyncConfig() - end - - return Config(lith=lith, server=server, storage=storage, sync=sync) -end diff --git a/lithoglyph/analytics/src/lith_client.jl b/lithoglyph/analytics/src/lith_client.jl deleted file mode 100644 index 79383e7f..00000000 --- a/lithoglyph/analytics/src/lith_client.jl +++ /dev/null @@ -1,160 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -HTTP client for Lith API -""" - -""" -Lith HTTP client -""" -struct LithClient - base_url::String - timeout::Int - - function LithClient(base_url::String; timeout::Int=30) - # Remove trailing slash - url = rstrip(base_url, '/') - new(url, timeout) - end -end - -""" -Document from Lith with provenance -""" -struct Document - id::String - data::Dict{String,Any} - provenance::Union{Dict{String,Any},Nothing} -end - -""" - fetch_collection(client::LithClient, collection::String) -> Vector{Document} - -Fetch all documents from a Lith collection. -""" -function fetch_collection(client::LithClient, collection::String)::Vector{Document} - url = "$(client.base_url)/collections/$collection/documents" - - @info "Fetching documents from Lith" url - - response = HTTP.get(url; readtimeout=client.timeout) - - if response.status != 200 - error("Lith API error: $(response.status)") - end - - docs_json = JSON3.read(String(response.body)) - - documents = Document[] - for doc in docs_json - push!(documents, Document( - string(doc.id), - convert(Dict{String,Any}, doc.data), - haskey(doc, :provenance) ? convert(Dict{String,Any}, doc.provenance) : nothing - )) - end - - @info "Fetched documents" count=length(documents) collection - - return documents -end - -""" - fetch_document(client::LithClient, collection::String, id::String) -> Document - -Fetch a single document by ID. -""" -function fetch_document(client::LithClient, collection::String, id::String)::Document - url = "$(client.base_url)/collections/$collection/documents/$id" - - response = HTTP.get(url; readtimeout=client.timeout) - - if response.status != 200 - error("Document not found: $id") - end - - doc = JSON3.read(String(response.body)) - - return Document( - string(doc.id), - convert(Dict{String,Any}, doc.data), - haskey(doc, :provenance) ? convert(Dict{String,Any}, doc.provenance) : nothing - ) -end - -""" - health_check(client::LithClient) -> Bool - -Check if Lith API is reachable. -""" -function health_check(client::LithClient)::Bool - try - url = "$(client.base_url)/health" - response = HTTP.get(url; readtimeout=5) - return response.status == 200 - catch - return false - end -end - -""" - extract_prompt_scores(doc::Document) -> Union{NamedTuple,Nothing} - -Extract PROMPT scores from a document if present. -Returns (provenance, replicability, objective, methodology, publication, transparency). -""" -function extract_prompt_scores(doc::Document) - data = doc.data - - # Try to find PROMPT scores in data - prompt = get(data, "prompt_scores", get(data, "prompt", nothing)) - - if isnothing(prompt) - return nothing - end - - if prompt isa Dict - return ( - provenance = get(prompt, "provenance", get(prompt, "P", missing)), - replicability = get(prompt, "replicability", get(prompt, "R", missing)), - objective = get(prompt, "objective", get(prompt, "O", missing)), - methodology = get(prompt, "methodology", get(prompt, "M", missing)), - publication = get(prompt, "publication", get(prompt, "P2", missing)), - transparency = get(prompt, "transparency", get(prompt, "T", missing)) - ) - end - - return nothing -end - -""" - extract_timestamp(doc::Document) -> Union{DateTime,Nothing} - -Extract timestamp from document provenance or data. -""" -function extract_timestamp(doc::Document) - # Try provenance first - if !isnothing(doc.provenance) - created_at = get(doc.provenance, "created_at", nothing) - if !isnothing(created_at) - try - return DateTime(created_at[1:19], "yyyy-mm-ddTHH:MM:SS") - catch - # Try other formats - end - end - end - - # Try data fields - for field in ["created_at", "timestamp", "date", "created"] - ts = get(doc.data, field, nothing) - if !isnothing(ts) && ts isa String - try - return DateTime(ts[1:19], "yyyy-mm-ddTHH:MM:SS") - catch - continue - end - end - end - - return nothing -end diff --git a/lithoglyph/analytics/src/main.jl b/lithoglyph/analytics/src/main.jl deleted file mode 100644 index b24fce95..00000000 --- a/lithoglyph/analytics/src/main.jl +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env julia -# SPDX-License-Identifier: MPL-2.0 -""" -Lith-Analytics entry point -""" - -using Pkg -Pkg.activate(@__DIR__ |> dirname) - -using LithAnalytics - -function main() - # Parse arguments - config_path = "config.toml" - - for (i, arg) in enumerate(ARGS) - if arg == "--config" && i < length(ARGS) - config_path = ARGS[i + 1] - elseif arg == "--help" - println(""" -Lith-Analytics - OLAP analytics layer for Lith - -Usage: - julia src/main.jl [options] - -Options: - --config PATH Path to config file (default: config.toml) - --help Show this help message - """) - return - end - end - - # Load configuration - config = load_config(config_path) - - # Start server - serve(config) -end - -main() diff --git a/lithoglyph/analytics/test/runtests.jl b/lithoglyph/analytics/test/runtests.jl deleted file mode 100644 index 6c689600..00000000 --- a/lithoglyph/analytics/test/runtests.jl +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -""" -Lith-Analytics test suite -""" - -using Test -using LithAnalytics -using DataFrames -using Dates - -@testset "LithAnalytics" begin - @testset "Config" begin - # Test default config - config = Config() - @test config.lith.api_url == "http://localhost:8080" - @test config.server.port == 8082 - @test config.storage.data_dir == "./data" - end - - @testset "ColumnarStore" begin - # Test store creation - mktempdir() do tmpdir - store = ColumnarStore(tmpdir) - @test isempty(store.collections) - - # Test stats on empty store - s = stats(store) - @test isempty(s) - end - end - - @testset "PROMPT extraction" begin - # Test document with PROMPT scores - doc = Document( - "test-001", - Dict{String,Any}( - "title" => "Test Document", - "prompt_scores" => Dict{String,Any}( - "provenance" => 0.8, - "replicability" => 0.7, - "objective" => 0.9, - "methodology" => 0.85, - "publication" => 0.6, - "transparency" => 0.95 - ) - ), - nothing - ) - - prompt = extract_prompt_scores(doc) - @test !isnothing(prompt) - @test prompt.provenance == 0.8 - @test prompt.transparency == 0.95 - - # Test document without PROMPT scores - doc_no_prompt = Document( - "test-002", - Dict{String,Any}("title" => "No PROMPT"), - nothing - ) - - @test isnothing(extract_prompt_scores(doc_no_prompt)) - end - - @testset "Timestamp extraction" begin - # Test provenance timestamp - doc = Document( - "test-001", - Dict{String,Any}(), - Dict{String,Any}("created_at" => "2025-01-16T12:00:00Z") - ) - - ts = extract_timestamp(doc) - @test !isnothing(ts) - @test year(ts) == 2025 - - # Test data timestamp - doc_data_ts = Document( - "test-002", - Dict{String,Any}("timestamp" => "2025-01-15T10:00:00"), - nothing - ) - - ts2 = extract_timestamp(doc_data_ts) - @test !isnothing(ts2) - @test day(ts2) == 15 - end -end diff --git a/lithoglyph/api/README.adoc b/lithoglyph/api/README.adoc deleted file mode 100644 index 953c6493..00000000 --- a/lithoglyph/api/README.adoc +++ /dev/null @@ -1,203 +0,0 @@ -= Lith API Server -:toc: - -Multi-protocol API server for Lith operations. - -== Overview - -The Lith API server provides three interfaces: - -* **REST API** - JSON over HTTP/1.1 (OpenAPI 3.1) -* **gRPC** - Protocol Buffers over HTTP/2 -* **GraphQL** - SDL with queries, mutations, and subscriptions - -== Quick Start - -[source,bash] ----- -# Build -cd api -zig build - -# Run (defaults to localhost:8080) -./zig-out/bin/lith-server - -# Or with custom config -LITH_HOST=0.0.0.0 LITH_PORT=9000 ./zig-out/bin/lith-server ----- - -== Configuration - -Environment variables: - -[cols="2,3,1"] -|=== -| Variable | Description | Default - -| `LITH_HOST` | Listen address | `127.0.0.1` -| `LITH_PORT` | Listen port | `8080` -| `LITH_DB_PATH` | Database file path | `./lith.dat` -| `LITH_JWT_SECRET` | JWT signing secret | (none) -| `LITH_API_KEY_HEADER` | API key header name | `X-API-Key` -| `LITH_REQUIRE_AUTH` | Require authentication | `false` -| `LITH_TLS_CERT` | TLS certificate path | (none) -| `LITH_TLS_KEY` | TLS private key path | (none) -|=== - -== API Specifications - -* **OpenAPI**: `spec/openapi.yaml` -* **Protocol Buffers**: `proto/lith.proto` -* **GraphQL SDL**: `graphql/schema.graphql` - -== Endpoints - -=== REST API (`/v1/`) - -[source] ----- -POST /v1/query # Execute GQL query -GET /v1/collections # List collections -POST /v1/collections # Create collection -GET /v1/collections/{name} # Get collection schema -DELETE /v1/collections/{name} # Drop collection -GET /v1/journal # Query journal entries -POST /v1/normalize/discover # Discover FDs -POST /v1/normalize/analyze # Analyze normal form -POST /v1/migrate/start # Start migration -POST /v1/migrate/{id}/shadow -POST /v1/migrate/{id}/commit -POST /v1/migrate/{id}/abort -GET /v1/health # Health check -GET /v1/metrics # Prometheus metrics ----- - -=== gRPC (`/grpc/`) - -[source,protobuf] ----- -service Lith { - rpc Query(QueryRequest) returns (QueryResponse); - rpc ListCollections(Empty) returns (ListCollectionsResponse); - rpc CreateCollection(...) returns (Collection); - rpc GetJournal(...) returns (stream JournalEntry); - rpc DiscoverDependencies(...) returns (DiscoverResponse); - rpc StartMigration(...) returns (MigrationResponse); - rpc Health(...) returns (HealthResponse); -} ----- - -=== GraphQL (`/graphql`) - -[source,graphql] ----- -type Query { - collections: [Collection!]! - journal(since: Int): [JournalEntry!]! - query(gql: String!): QueryResult! - health: Health! -} - -type Mutation { - createCollection(...): Collection! - execute(gql: String!): MutationResult! - startMigration(...): Migration! -} - -type Subscription { - journalStream: JournalEntry! -} ----- - -GET `/graphql` returns GraphiQL UI for interactive exploration. - -== Authentication - -=== API Key - -[source,bash] ----- -curl -H "X-API-Key: your-key" http://localhost:8080/v1/collections ----- - -=== JWT Bearer Token - -[source,bash] ----- -curl -H "Authorization: Bearer eyJ..." http://localhost:8080/v1/collections ----- - -=== mTLS (gRPC) - -For gRPC, mTLS can be configured for client certificate authentication. - -== Provenance - -All mutating operations support provenance tracking: - -[source,json] ----- -{ - "gql": "INSERT INTO articles VALUES {...}", - "provenance": { - "actor": "editor@news.org", - "rationale": "New article creation" - } -} ----- - -Or via headers: - -[source,bash] ----- -curl -H "X-Provenance-Actor: editor@news.org" \ - -H "X-Provenance-Rationale: New article" \ - ... ----- - -== Metrics - -Prometheus metrics available at `/v1/metrics`: - -* `lith_requests_total` - Total requests -* `lith_requests_by_status` - Requests by HTTP status -* `lith_requests_by_protocol` - Requests by protocol (REST/gRPC/GraphQL) -* `lith_request_duration_seconds` - Request latency histogram -* `lith_active_connections` - Current connections -* `lith_active_migrations` - Active migrations -* `lith_uptime_seconds` - Server uptime - -== Development - -[source,bash] ----- -# Run tests -zig build test - -# Build release -zig build -Doptimize=ReleaseFast ----- - -== Directory Structure - -[source] ----- -api/ -├── build.zig # Zig build configuration -├── README.adoc # This file -├── spec/ -│ └── openapi.yaml # OpenAPI 3.1 specification -├── proto/ -│ └── lith.proto # Protocol Buffer definitions -├── graphql/ -│ └── schema.graphql # GraphQL SDL schema -└── src/ - ├── main.zig # Server entry point - ├── config.zig # Configuration loading - ├── router.zig # HTTP routing - ├── rest.zig # REST API handlers - ├── grpc.zig # gRPC handlers - ├── graphql.zig # GraphQL handlers - ├── metrics.zig # Prometheus metrics - └── auth.zig # Authentication ----- diff --git a/lithoglyph/api/build.zig b/lithoglyph/api/build.zig deleted file mode 100644 index 2c0214b5..00000000 --- a/lithoglyph/api/build.zig +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Build Configuration - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Main server executable - const exe_mod = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const exe = b.addExecutable(.{ - .name = "lithoglyph-server", - .root_module = exe_mod, - }); - - // Link libc for networking - exe.linkLibC(); - - // Link bridge library (from core-zig build output) - exe.addLibraryPath(b.path("../core-zig/zig-out/lib")); - exe.linkSystemLibrary("lith_bridge"); - - b.installArtifact(exe); - - // Run command - const run_cmd = b.addRunArtifact(exe); - run_cmd.step.dependOn(b.getInstallStep()); - - if (b.args) |args| { - run_cmd.addArgs(args); - } - - const run_step = b.step("run", "Run the Lith API server"); - run_step.dependOn(&run_cmd.step); - - // Tests - const test_mod = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - const unit_tests = b.addTest(.{ - .root_module = test_mod, - }); - - const run_unit_tests = b.addRunArtifact(unit_tests); - - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_unit_tests.step); - - // Module tests - const modules = [_][]const u8{ - "src/config.zig", - "src/router.zig", - "src/rest.zig", - "src/grpc.zig", - "src/graphql.zig", - "src/metrics.zig", - "src/auth.zig", - "src/bridge_client.zig", - "src/websocket.zig", - "src/integration_tests.zig", - }; - - for (modules) |mod| { - const mod_mod = b.createModule(.{ - .root_source_file = b.path(mod), - .target = target, - .optimize = optimize, - }); - const mod_test = b.addTest(.{ - .root_module = mod_mod, - }); - const run_mod_test = b.addRunArtifact(mod_test); - test_step.dependOn(&run_mod_test.step); - } -} diff --git a/lithoglyph/api/graphql/bofig_ingest.graphql b/lithoglyph/api/graphql/bofig_ingest.graphql deleted file mode 100644 index 28fb81c3..00000000 --- a/lithoglyph/api/graphql/bofig_ingest.graphql +++ /dev/null @@ -1,403 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# L6 — GraphQL schema extension for Bofig evidence ingest. -# Supports single and batch ingest from Docudactyl Cap'n Proto payloads, -# with PROMPT scores, entity extraction, and financial data. - -# ============================================================================= -# Extend Root Types -# ============================================================================= - -extend type Mutation { - """Ingest a single evidence record into the Bofig collections.""" - ingestEvidence(input: IngestEvidenceInput!): IngestResult! - - """Batch ingest multiple evidence records with shared provenance.""" - batchIngestEvidence(input: BatchIngestInput!): BatchIngestResult! -} - -extend type Subscription { - """Stream progress updates for a batch ingest run.""" - ingestProgress(runId: ID!): IngestProgress! -} - -# ============================================================================= -# Input Types -# ============================================================================= - -"""A single evidence record for ingest, typically from a Docudactyl pipeline.""" -input IngestEvidenceInput { - """Investigation this evidence belongs to.""" - investigationId: String! - - """Human-readable title for the evidence.""" - title: String! - - """Classification of evidence source type.""" - evidenceType: EvidenceType! - - """Extracted text content (OCR'd if from scanned document).""" - contentText: String - - """URL or file path to original source document.""" - urlSource: String - - """SHA-256 hash for deduplication and integrity verification.""" - sha256Hash: String! - - """Perceptual hash for near-duplicate image detection.""" - perceptualHash: String - - """Tesseract OCR confidence score (0-100), if applicable.""" - ocrConfidence: Int - - """ISO 639-1 language code of primary content.""" - language: String - - """Date of the original document (not ingestion date).""" - documentDate: DateTime - - """NER-extracted entity names.""" - entitiesExtracted: [String!] - - """Auto-extracted keywords from content.""" - keywords: [String!] - - """Redaction status of the document.""" - redactionStatus: RedactionStatus - - """Number of detected redaction annotations.""" - redactionCount: Int - - """Docudactyl pipeline run identifier for provenance chain.""" - extractionRunId: String - - """Dublin Core metadata as JSON string.""" - dublinCoreMetadata: JSON - - """Access control classification.""" - sensitivityLevel: SensitivityLevel - - """PROMPT dimension scores for research-grade data quality.""" - promptScores: PromptScoresInput - - """Analyst notes and observations.""" - notes: String - - """Entities to create or link (co-reference resolution candidates).""" - entities: [IngestEntityInput!] - - """Financial transactions extracted from this evidence.""" - financialTransactions: [IngestFinancialTransactionInput!] - - """Claims extracted from this evidence.""" - claims: [IngestClaimInput!] - - """Provenance for this ingest operation.""" - provenance: ProvenanceInput! -} - -"""PROMPT dimension scores (each 0-100).""" -input PromptScoresInput { - provenance: Int! - replicability: Int! - objective: Int! - methodology: Int! - publication: Int! - transparency: Int! -} - -"""An entity extracted from evidence, for creation or co-reference linking.""" -input IngestEntityInput { - """Candidate primary name.""" - primaryName: String! - - """Entity type classification.""" - entityType: EntityTypeEnum! - - """Known aliases for this entity.""" - aliases: [String!] - - """Brief description of entity role.""" - description: String - - """If this entity matches an existing entity, provide its ID for linking.""" - existingEntityId: String - - """Jaro-Winkler similarity score if matched to existing entity.""" - matchScore: Float -} - -"""A financial transaction extracted from evidence.""" -input IngestFinancialTransactionInput { - """Entity ID or name of the sender/payer.""" - sourceEntity: String! - - """Entity ID or name of the receiver/payee.""" - destinationEntity: String! - - """Transaction amount in original currency.""" - amount: BigInt! - - """Currency code.""" - currency: CurrencyCode! - - """Date of the transaction.""" - transactionDate: DateTime! - - """Payment instrument or method.""" - instrument: FinancialInstrument - - """Entity ID of any intermediary.""" - intermediaryEntity: String - - """Source account identifier (masked).""" - sourceAccount: String - - """Destination account identifier.""" - destinationAccount: String - - """Automated anomaly detection flags.""" - anomalyFlags: [AnomalyFlag!] - - """Notes on this transaction.""" - notes: String -} - -"""A claim extracted from evidence.""" -input IngestClaimInput { - """The specific factual assertion.""" - claimText: String! - - """Claim type classification.""" - claimType: ClaimType! - - """Confidence in claim accuracy (0-100).""" - confidence: Int! - - """Entity ID of person who made this claim (for testimony).""" - speakerId: String - - """Date the claimed event occurred.""" - claimDate: DateTime - - """Date the claim was made.""" - assertionDate: DateTime - - """PROMPT scores for this claim.""" - promptScores: PromptScoresInput -} - -"""Batch ingest input wrapping multiple evidence records.""" -input BatchIngestInput { - """Investigation this batch belongs to.""" - investigationId: String! - - """Docudactyl extraction run ID for the entire batch.""" - extractionRunId: String! - - """Evidence records to ingest.""" - records: [IngestEvidenceInput!]! - - """Global provenance for the batch operation.""" - provenance: ProvenanceInput! - - """Skip records that already exist (by sha256Hash).""" - deduplicateByHash: Boolean = true - - """Continue on individual record failure.""" - continueOnError: Boolean = true -} - -# ============================================================================= -# Enum Types -# ============================================================================= - -enum EvidenceType { - COURT_FILING - DEPOSITION - TESTIMONY - FLIGHT_LOG - FINANCIAL_RECORD - COMMUNICATION - PHOTOGRAPH - VIDEO - OFFICIAL_STATISTICS - NEWS_REPORT - DOCUMENT - DATASET - INTERVIEW - AFFIDAVIT - SUBPOENA - OTHER -} - -enum RedactionStatus { - CLEAN - REDACTED - PARTIALLY_RECOVERED - FULLY_RECOVERED - NOT_APPLICABLE -} - -enum SensitivityLevel { - PUBLIC - RESTRICTED - SEALED - SOURCE_PROTECTED -} - -enum EntityTypeEnum { - PERSON - ORGANIZATION - LOCATION - ACCOUNT - VESSEL - AIRCRAFT -} - -enum CurrencyCode { - USD - GBP - EUR - CHF - JPY - CAD - AUD - OTHER -} - -enum FinancialInstrument { - WIRE_TRANSFER - CHECK - CASH - CREDIT_CARD - CRYPTO - TRUST_PAYMENT - SHELL_COMPANY - OTHER -} - -enum AnomalyFlag { - ROUND_NUMBER - STRUCTURING - UNUSUAL_TIMING - NEW_COUNTERPARTY - HIGH_VALUE - CROSS_BORDER -} - -enum ClaimType { - FACTUAL - TEMPORAL - FINANCIAL - TESTIMONIAL - DOCUMENTARY - CIRCUMSTANTIAL - CORROBORATIVE - CONTRADICTORY -} - -# ============================================================================= -# Result Types -# ============================================================================= - -"""Result of a single evidence ingest operation.""" -type IngestResult { - """Whether the ingest succeeded.""" - success: Boolean! - - """ID of the created evidence record.""" - evidenceId: String - - """Journal sequence number for the operation.""" - journalSeq: BigInt - - """Number of entities created or linked.""" - entitiesProcessed: Int! - - """Number of financial transactions created.""" - transactionsProcessed: Int! - - """Number of claims created.""" - claimsProcessed: Int! - - """Errors encountered (empty on full success).""" - errors: [IngestError!]! -} - -"""Result of a batch ingest operation.""" -type BatchIngestResult { - """Unique identifier for this ingest run.""" - runId: ID! - - """Total records submitted.""" - totalSubmitted: Int! - - """Records successfully imported.""" - importedCount: Int! - - """Records skipped (e.g., duplicate hash).""" - skippedCount: Int! - - """Records that failed to import.""" - failedCount: Int! - - """Per-record results (in submission order).""" - results: [IngestResult!]! - - """Timing information for the batch.""" - timing: Timing -} - -"""Progress update for a batch ingest run.""" -type IngestProgress { - """Ingest run identifier.""" - runId: ID! - - """Current phase of the ingest.""" - phase: IngestPhase! - - """Progress as a fraction (0.0 to 1.0).""" - progress: Float! - - """Number of records processed so far.""" - processedCount: Int! - - """Total records in the batch.""" - totalCount: Int! - - """Human-readable status message.""" - message: String! - - """Current record being processed (if applicable).""" - currentRecord: String -} - -"""An error encountered during ingest.""" -type IngestError { - """Zero-based index of the record that failed.""" - recordIndex: Int - - """Error code.""" - code: String! - - """Human-readable error message.""" - message: String! - - """SHA-256 hash of the failed record (for identification).""" - sha256Hash: String -} - -enum IngestPhase { - VALIDATING - DEDUPLICATING - INSERTING_EVIDENCE - RESOLVING_ENTITIES - INSERTING_TRANSACTIONS - INSERTING_CLAIMS - CREATING_RELATIONSHIPS - COMPLETE - FAILED -} diff --git a/lithoglyph/api/graphql/schema.graphql b/lithoglyph/api/graphql/schema.graphql deleted file mode 100644 index cb24d620..00000000 --- a/lithoglyph/api/graphql/schema.graphql +++ /dev/null @@ -1,477 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Lith GraphQL API - Schema Definition Language - -# ============================================================================= -# Scalars -# ============================================================================= - -scalar DateTime -scalar JSON -scalar BigInt - -# ============================================================================= -# Root Types -# ============================================================================= - -type Query { - """List all collections""" - collections(limit: Int = 100, offset: Int = 0): CollectionConnection! - - """Get a specific collection by name""" - collection(name: String!): Collection - - """Query journal entries""" - journal( - since: BigInt - limit: Int = 100 - collection: String - actor: String - ): JournalConnection! - - """Execute an GQL query""" - query( - gql: String! - provenance: ProvenanceInput - ): QueryResult! - - """Explain an GQL query without executing""" - explain( - gql: String! - analyze: Boolean = false - verbose: Boolean = false - ): ExplainResult! - - """Get migration status""" - migration(id: ID!): Migration - - """List active migrations""" - activeMigrations: [Migration!]! - - """Health check""" - health: Health! -} - -type Mutation { - """Create a new collection""" - createCollection( - input: CreateCollectionInput! - ): Collection! - - """Drop a collection""" - dropCollection( - name: String! - provenance: ProvenanceInput! - ): DropCollectionResult! - - """Execute an GQL mutation (INSERT, UPDATE, DELETE)""" - execute( - gql: String! - provenance: ProvenanceInput! - ): MutationResult! - - """Discover functional dependencies in a collection""" - discoverDependencies( - collection: String! - sampleSize: Int = 1000 - confidenceThreshold: Float = 0.95 - ): DiscoverResult! - - """Analyze normal form of a collection""" - analyzeNormalForm( - collection: String! - ): AnalyzeResult! - - """Start a schema migration""" - startMigration( - input: MigrationStartInput! - ): Migration! - - """Advance migration to shadow phase""" - advanceToShadow( - migrationId: ID! - provenance: ProvenanceInput! - ): Migration! - - """Commit a migration""" - commitMigration( - migrationId: ID! - provenance: ProvenanceInput! - ): Migration! - - """Abort a migration""" - abortMigration( - migrationId: ID! - provenance: ProvenanceInput! - ): Migration! -} - -type Subscription { - """Stream new journal entries""" - journalStream( - collection: String - since: BigInt - ): JournalEntry! - - """Stream query results for large result sets""" - queryStream( - gql: String! - provenance: ProvenanceInput - ): JSON! - - """Migration progress updates""" - migrationProgress( - migrationId: ID! - ): MigrationProgress! -} - -# ============================================================================= -# Collection Types -# ============================================================================= - -type Collection { - name: String! - type: CollectionType! - schema: Schema - documentCount: BigInt! - createdAt: DateTime! - normalForm: NormalForm - - """Sample documents from this collection""" - documents(limit: Int = 10, offset: Int = 0): DocumentConnection! - - """Functional dependencies discovered in this collection""" - functionalDependencies: [FunctionalDependency!]! -} - -type CollectionConnection { - edges: [CollectionEdge!]! - pageInfo: PageInfo! - totalCount: Int! -} - -type CollectionEdge { - node: Collection! - cursor: String! -} - -enum CollectionType { - DOCUMENT - EDGE -} - -type Schema { - fields: [Field!]! - constraints: [Constraint!]! -} - -type Field { - name: String! - type: FieldType! - nullable: Boolean! -} - -enum FieldType { - STRING - INTEGER - FLOAT - BOOLEAN - DATE - DATETIME - JSON - BLOB -} - -type Constraint { - type: ConstraintType! - fields: [String!]! - expression: String -} - -enum ConstraintType { - PRIMARY_KEY - UNIQUE - FOREIGN_KEY - CHECK - FUNCTIONAL_DEPENDENCY -} - -type DocumentConnection { - edges: [DocumentEdge!]! - pageInfo: PageInfo! - totalCount: Int! -} - -type DocumentEdge { - node: JSON! - cursor: String! -} - -# ============================================================================= -# Query Types -# ============================================================================= - -type QueryResult { - rows: [JSON!]! - rowCount: Int! - journalSeq: BigInt - timing: Timing -} - -type MutationResult { - affectedCount: Int! - journalSeq: BigInt! - timing: Timing -} - -type ExplainResult { - plan: QueryPlan! - timing: Timing - verboseOutput: String -} - -type QueryPlan { - steps: [PlanStep!]! - estimatedCost: Float! - rationale: String -} - -type PlanStep { - type: StepType! - collection: String - details: JSON -} - -enum StepType { - SCAN - FILTER - PROJECT - LIMIT - TRAVERSE - INSERT - UPDATE - DELETE -} - -type Timing { - parseMs: Float! - planMs: Float! - executeMs: Float! - totalMs: Float! -} - -type DropCollectionResult { - success: Boolean! - journalSeq: BigInt -} - -# ============================================================================= -# Journal Types -# ============================================================================= - -type JournalEntry { - seq: BigInt! - timestamp: DateTime! - operation: OperationType! - collection: String - documentId: String - before: JSON - after: JSON - provenance: Provenance - inverse: String -} - -type JournalConnection { - edges: [JournalEdge!]! - pageInfo: PageInfo! - hasMore: Boolean! - nextSeq: BigInt -} - -type JournalEdge { - node: JournalEntry! - cursor: String! -} - -enum OperationType { - INSERT - UPDATE - DELETE - CREATE_COLLECTION - DROP_COLLECTION - MIGRATION -} - -# ============================================================================= -# Normalizer Types -# ============================================================================= - -type FunctionalDependency { - determinant: [String!]! - dependent: String! - confidence: Float! - tier: ConfidenceTier! -} - -enum ConfidenceTier { - HIGH - MEDIUM - LOW -} - -type DiscoverResult { - collection: String! - functionalDependencies: [FunctionalDependency!]! - candidateKeys: [[String!]!]! -} - -type AnalyzeResult { - collection: String! - currentForm: NormalForm! - violations: [Violation!]! - recommendations: [Recommendation!]! -} - -enum NormalForm { - NF_1NF - NF_2NF - NF_3NF - NF_BCNF - UNKNOWN -} - -type Violation { - type: ViolationType! - description: String! - affectedFields: [String!]! -} - -enum ViolationType { - PARTIAL_DEPENDENCY - TRANSITIVE_DEPENDENCY - BCNF_VIOLATION -} - -type Recommendation { - action: RecommendationAction! - description: String! - targetForm: NormalForm - migrationSteps: [String!]! -} - -enum RecommendationAction { - DECOMPOSE - ADD_CONSTRAINT - DENORMALIZE -} - -# ============================================================================= -# Migration Types -# ============================================================================= - -type Migration { - id: ID! - collection: String! - phase: MigrationPhase! - targetForm: NormalForm! - startedAt: DateTime! - narrative: String! - rewriteRules: [RewriteRule!]! -} - -enum MigrationPhase { - ANNOUNCE - SHADOW - COMMIT - ABORTED - COMPLETE -} - -type RewriteRule { - oldPath: String! - newPath: String! - transformation: String -} - -type MigrationProgress { - migrationId: ID! - phase: MigrationPhase! - progress: Float! - message: String! -} - -# ============================================================================= -# Health Types -# ============================================================================= - -type Health { - status: HealthStatus! - version: String! - uptimeSeconds: BigInt! - checks: [HealthCheck!]! -} - -enum HealthStatus { - HEALTHY - DEGRADED - UNHEALTHY -} - -type HealthCheck { - name: String! - status: CheckStatus! -} - -enum CheckStatus { - PASS - FAIL -} - -# ============================================================================= -# Input Types -# ============================================================================= - -input ProvenanceInput { - actor: String! - rationale: String! -} - -input CreateCollectionInput { - name: String! - type: CollectionType = DOCUMENT - schema: SchemaInput - provenance: ProvenanceInput! -} - -input SchemaInput { - fields: [FieldInput!]! - constraints: [ConstraintInput!] -} - -input FieldInput { - name: String! - type: FieldType! - nullable: Boolean = true -} - -input ConstraintInput { - type: ConstraintType! - fields: [String!]! - expression: String -} - -input MigrationStartInput { - collection: String! - targetForm: NormalForm! - provenance: ProvenanceInput! -} - -# ============================================================================= -# Pagination Types -# ============================================================================= - -type PageInfo { - hasNextPage: Boolean! - hasPreviousPage: Boolean! - startCursor: String - endCursor: String -} diff --git a/lithoglyph/api/proto/bofig_ingest.proto b/lithoglyph/api/proto/bofig_ingest.proto deleted file mode 100644 index a5fed649..00000000 --- a/lithoglyph/api/proto/bofig_ingest.proto +++ /dev/null @@ -1,311 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// L6 — gRPC protocol buffer definitions for Bofig evidence ingest. -// Supports single and batch ingest from Docudactyl Cap'n Proto payloads, -// with PROMPT scores, entity extraction, and financial data. - -syntax = "proto3"; - -package lith.bofig.v1; - -option go_package = "github.com/hyperpolymath/lith/api/gen/go/lith/bofig/v1"; - -import "google/protobuf/timestamp.proto"; -import "google/protobuf/struct.proto"; - -// ============================================================================= -// BofigIngest Service -// ============================================================================= - -service BofigIngest { - // Ingest a single evidence record into the Bofig collections. - rpc IngestEvidence(IngestEvidenceRequest) returns (IngestEvidenceResponse); - - // Batch ingest multiple evidence records with shared provenance. - rpc BatchIngest(BatchIngestRequest) returns (BatchIngestResponse); - - // Get the status of a batch ingest run. - rpc IngestStatus(IngestStatusRequest) returns (IngestStatusResponse); - - // Stream progress updates for a batch ingest run. - rpc IngestProgressStream(IngestProgressRequest) returns (stream IngestProgressUpdate); -} - -// ============================================================================= -// Common Types -// ============================================================================= - -message Provenance { - string actor = 1; - string rationale = 2; -} - -// PROMPT dimension scores (each 0-100). -message PromptScores { - int32 provenance = 1; - int32 replicability = 2; - int32 objective = 3; - int32 methodology = 4; - int32 publication = 5; - int32 transparency = 6; -} - -// ============================================================================= -// Enums -// ============================================================================= - -enum EvidenceType { - EVIDENCE_TYPE_UNSPECIFIED = 0; - EVIDENCE_TYPE_COURT_FILING = 1; - EVIDENCE_TYPE_DEPOSITION = 2; - EVIDENCE_TYPE_TESTIMONY = 3; - EVIDENCE_TYPE_FLIGHT_LOG = 4; - EVIDENCE_TYPE_FINANCIAL_RECORD = 5; - EVIDENCE_TYPE_COMMUNICATION = 6; - EVIDENCE_TYPE_PHOTOGRAPH = 7; - EVIDENCE_TYPE_VIDEO = 8; - EVIDENCE_TYPE_OFFICIAL_STATISTICS = 9; - EVIDENCE_TYPE_NEWS_REPORT = 10; - EVIDENCE_TYPE_DOCUMENT = 11; - EVIDENCE_TYPE_DATASET = 12; - EVIDENCE_TYPE_INTERVIEW = 13; - EVIDENCE_TYPE_AFFIDAVIT = 14; - EVIDENCE_TYPE_SUBPOENA = 15; - EVIDENCE_TYPE_OTHER = 16; -} - -enum RedactionStatus { - REDACTION_STATUS_UNSPECIFIED = 0; - REDACTION_STATUS_CLEAN = 1; - REDACTION_STATUS_REDACTED = 2; - REDACTION_STATUS_PARTIALLY_RECOVERED = 3; - REDACTION_STATUS_FULLY_RECOVERED = 4; - REDACTION_STATUS_NOT_APPLICABLE = 5; -} - -enum SensitivityLevel { - SENSITIVITY_LEVEL_UNSPECIFIED = 0; - SENSITIVITY_LEVEL_PUBLIC = 1; - SENSITIVITY_LEVEL_RESTRICTED = 2; - SENSITIVITY_LEVEL_SEALED = 3; - SENSITIVITY_LEVEL_SOURCE_PROTECTED = 4; -} - -enum EntityType { - ENTITY_TYPE_UNSPECIFIED = 0; - ENTITY_TYPE_PERSON = 1; - ENTITY_TYPE_ORGANIZATION = 2; - ENTITY_TYPE_LOCATION = 3; - ENTITY_TYPE_ACCOUNT = 4; - ENTITY_TYPE_VESSEL = 5; - ENTITY_TYPE_AIRCRAFT = 6; -} - -enum CurrencyCode { - CURRENCY_CODE_UNSPECIFIED = 0; - CURRENCY_CODE_USD = 1; - CURRENCY_CODE_GBP = 2; - CURRENCY_CODE_EUR = 3; - CURRENCY_CODE_CHF = 4; - CURRENCY_CODE_JPY = 5; - CURRENCY_CODE_CAD = 6; - CURRENCY_CODE_AUD = 7; - CURRENCY_CODE_OTHER = 8; -} - -enum FinancialInstrument { - FINANCIAL_INSTRUMENT_UNSPECIFIED = 0; - FINANCIAL_INSTRUMENT_WIRE_TRANSFER = 1; - FINANCIAL_INSTRUMENT_CHECK = 2; - FINANCIAL_INSTRUMENT_CASH = 3; - FINANCIAL_INSTRUMENT_CREDIT_CARD = 4; - FINANCIAL_INSTRUMENT_CRYPTO = 5; - FINANCIAL_INSTRUMENT_TRUST_PAYMENT = 6; - FINANCIAL_INSTRUMENT_SHELL_COMPANY = 7; - FINANCIAL_INSTRUMENT_OTHER = 8; -} - -enum AnomalyFlag { - ANOMALY_FLAG_UNSPECIFIED = 0; - ANOMALY_FLAG_ROUND_NUMBER = 1; - ANOMALY_FLAG_STRUCTURING = 2; - ANOMALY_FLAG_UNUSUAL_TIMING = 3; - ANOMALY_FLAG_NEW_COUNTERPARTY = 4; - ANOMALY_FLAG_HIGH_VALUE = 5; - ANOMALY_FLAG_CROSS_BORDER = 6; -} - -enum ClaimType { - CLAIM_TYPE_UNSPECIFIED = 0; - CLAIM_TYPE_FACTUAL = 1; - CLAIM_TYPE_TEMPORAL = 2; - CLAIM_TYPE_FINANCIAL = 3; - CLAIM_TYPE_TESTIMONIAL = 4; - CLAIM_TYPE_DOCUMENTARY = 5; - CLAIM_TYPE_CIRCUMSTANTIAL = 6; - CLAIM_TYPE_CORROBORATIVE = 7; - CLAIM_TYPE_CONTRADICTORY = 8; -} - -enum IngestPhase { - INGEST_PHASE_UNSPECIFIED = 0; - INGEST_PHASE_VALIDATING = 1; - INGEST_PHASE_DEDUPLICATING = 2; - INGEST_PHASE_INSERTING_EVIDENCE = 3; - INGEST_PHASE_RESOLVING_ENTITIES = 4; - INGEST_PHASE_INSERTING_TRANSACTIONS = 5; - INGEST_PHASE_INSERTING_CLAIMS = 6; - INGEST_PHASE_CREATING_RELATIONSHIPS = 7; - INGEST_PHASE_COMPLETE = 8; - INGEST_PHASE_FAILED = 9; -} - -// ============================================================================= -// Ingest Request Messages -// ============================================================================= - -// A single evidence record for ingest. -message EvidenceRecord { - string investigation_id = 1; - string title = 2; - EvidenceType evidence_type = 3; - string content_text = 4; - string url_source = 5; - string sha256_hash = 6; - string perceptual_hash = 7; - int32 ocr_confidence = 8; - string language = 9; - google.protobuf.Timestamp document_date = 10; - repeated string entities_extracted = 11; - repeated string keywords = 12; - RedactionStatus redaction_status = 13; - int32 redaction_count = 14; - string extraction_run_id = 15; - google.protobuf.Struct dublin_core_metadata = 16; - SensitivityLevel sensitivity_level = 17; - PromptScores prompt_scores = 18; - string notes = 19; - repeated EntityRecord entities = 20; - repeated FinancialTransactionRecord financial_transactions = 21; - repeated ClaimRecord claims = 22; -} - -// An entity extracted from evidence. -message EntityRecord { - string primary_name = 1; - EntityType entity_type = 2; - repeated string aliases = 3; - string description = 4; - // If matched to an existing entity, provide its ID. - string existing_entity_id = 5; - // Jaro-Winkler similarity score if matched. - double match_score = 6; -} - -// A financial transaction extracted from evidence. -message FinancialTransactionRecord { - string source_entity = 1; - string destination_entity = 2; - int64 amount = 3; - CurrencyCode currency = 4; - google.protobuf.Timestamp transaction_date = 5; - FinancialInstrument instrument = 6; - string intermediary_entity = 7; - string source_account = 8; - string destination_account = 9; - repeated AnomalyFlag anomaly_flags = 10; - string notes = 11; -} - -// A claim extracted from evidence. -message ClaimRecord { - string claim_text = 1; - ClaimType claim_type = 2; - int32 confidence = 3; - string speaker_id = 4; - google.protobuf.Timestamp claim_date = 5; - google.protobuf.Timestamp assertion_date = 6; - PromptScores prompt_scores = 7; -} - -// ============================================================================= -// RPC Request/Response Messages -// ============================================================================= - -message IngestEvidenceRequest { - EvidenceRecord record = 1; - Provenance provenance = 2; -} - -message IngestEvidenceResponse { - bool success = 1; - string evidence_id = 2; - int64 journal_seq = 3; - int32 entities_processed = 4; - int32 transactions_processed = 5; - int32 claims_processed = 6; - repeated IngestError errors = 7; -} - -message BatchIngestRequest { - string investigation_id = 1; - string extraction_run_id = 2; - repeated EvidenceRecord records = 3; - Provenance provenance = 4; - // Skip records that already exist (by sha256_hash). - bool deduplicate_by_hash = 5; - // Continue on individual record failure. - bool continue_on_error = 6; -} - -message BatchIngestResponse { - string run_id = 1; - int32 total_submitted = 2; - int32 imported_count = 3; - int32 skipped_count = 4; - int32 failed_count = 5; - repeated IngestEvidenceResponse results = 6; - Timing timing = 7; -} - -message IngestStatusRequest { - string run_id = 1; -} - -message IngestStatusResponse { - string run_id = 1; - IngestPhase phase = 2; - double progress = 3; - int32 processed_count = 4; - int32 total_count = 5; - string message = 6; -} - -message IngestProgressRequest { - string run_id = 1; -} - -message IngestProgressUpdate { - string run_id = 1; - IngestPhase phase = 2; - double progress = 3; - int32 processed_count = 4; - int32 total_count = 5; - string message = 6; - string current_record = 7; -} - -message IngestError { - int32 record_index = 1; - string code = 2; - string message = 3; - string sha256_hash = 4; -} - -message Timing { - double parse_ms = 1; - double plan_ms = 2; - double execute_ms = 3; - double total_ms = 4; -} diff --git a/lithoglyph/api/proto/lith.proto b/lithoglyph/api/proto/lith.proto deleted file mode 100644 index 4cdfb9fb..00000000 --- a/lithoglyph/api/proto/lith.proto +++ /dev/null @@ -1,406 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith gRPC API - Protocol Buffer Definitions - -syntax = "proto3"; - -package lith.v1; - -option go_package = "github.com/hyperpolymath/lith/api/gen/go/lith/v1"; - -import "google/protobuf/timestamp.proto"; -import "google/protobuf/struct.proto"; - -// ============================================================================= -// Lith Service -// ============================================================================= - -service Lith { - // Query Operations - rpc Query(QueryRequest) returns (QueryResponse); - rpc QueryStream(QueryRequest) returns (stream QueryRow); - rpc Explain(QueryRequest) returns (ExplainResponse); - - // Collection Operations - rpc CreateCollection(CreateCollectionRequest) returns (Collection); - rpc GetCollection(GetCollectionRequest) returns (Collection); - rpc ListCollections(ListCollectionsRequest) returns (ListCollectionsResponse); - rpc DropCollection(DropCollectionRequest) returns (DropCollectionResponse); - - // Journal Operations - rpc GetJournal(JournalRequest) returns (JournalResponse); - rpc StreamJournal(JournalStreamRequest) returns (stream JournalEntry); - - // Normalizer Operations - rpc DiscoverDependencies(DiscoverRequest) returns (DiscoverResponse); - rpc AnalyzeNormalForm(AnalyzeRequest) returns (AnalyzeResponse); - - // Migration Operations - rpc StartMigration(MigrationStartRequest) returns (MigrationResponse); - rpc AdvanceToShadow(MigrationAdvanceRequest) returns (MigrationResponse); - rpc CommitMigration(MigrationCommitRequest) returns (MigrationResponse); - rpc AbortMigration(MigrationAbortRequest) returns (MigrationResponse); - rpc GetMigrationStatus(MigrationStatusRequest) returns (MigrationResponse); - - // Health Operations - rpc Health(HealthRequest) returns (HealthResponse); -} - -// ============================================================================= -// Common Types -// ============================================================================= - -message Provenance { - string actor = 1; - string rationale = 2; -} - -message Timing { - double parse_ms = 1; - double plan_ms = 2; - double execute_ms = 3; - double total_ms = 4; -} - -// ============================================================================= -// Query Messages -// ============================================================================= - -message QueryRequest { - string gql = 1; - Provenance provenance = 2; - bool explain = 3; - bool analyze = 4; - bool verbose = 5; -} - -message QueryResponse { - repeated QueryRow rows = 1; - int64 row_count = 2; - int64 journal_seq = 3; - Timing timing = 4; -} - -message QueryRow { - google.protobuf.Struct data = 1; -} - -message ExplainResponse { - QueryPlan plan = 1; - Timing timing = 2; - string verbose_output = 3; -} - -message QueryPlan { - repeated PlanStep steps = 1; - double estimated_cost = 2; - string rationale = 3; -} - -message PlanStep { - StepType type = 1; - string collection = 2; - google.protobuf.Struct details = 3; -} - -enum StepType { - STEP_TYPE_UNSPECIFIED = 0; - STEP_TYPE_SCAN = 1; - STEP_TYPE_FILTER = 2; - STEP_TYPE_PROJECT = 3; - STEP_TYPE_LIMIT = 4; - STEP_TYPE_TRAVERSE = 5; - STEP_TYPE_INSERT = 6; - STEP_TYPE_UPDATE = 7; - STEP_TYPE_DELETE = 8; -} - -// ============================================================================= -// Collection Messages -// ============================================================================= - -message CreateCollectionRequest { - string name = 1; - CollectionType type = 2; - Schema schema = 3; - Provenance provenance = 4; -} - -message GetCollectionRequest { - string name = 1; -} - -message ListCollectionsRequest { - int32 limit = 1; - int32 offset = 2; -} - -message ListCollectionsResponse { - repeated Collection collections = 1; - int32 total = 2; -} - -message DropCollectionRequest { - string name = 1; - Provenance provenance = 2; -} - -message DropCollectionResponse { - bool success = 1; - int64 journal_seq = 2; -} - -message Collection { - string name = 1; - CollectionType type = 2; - Schema schema = 3; - int64 document_count = 4; - google.protobuf.Timestamp created_at = 5; - NormalForm normal_form = 6; -} - -enum CollectionType { - COLLECTION_TYPE_UNSPECIFIED = 0; - COLLECTION_TYPE_DOCUMENT = 1; - COLLECTION_TYPE_EDGE = 2; -} - -message Schema { - repeated Field fields = 1; - repeated Constraint constraints = 2; -} - -message Field { - string name = 1; - FieldType type = 2; - bool nullable = 3; -} - -enum FieldType { - FIELD_TYPE_UNSPECIFIED = 0; - FIELD_TYPE_STRING = 1; - FIELD_TYPE_INTEGER = 2; - FIELD_TYPE_FLOAT = 3; - FIELD_TYPE_BOOLEAN = 4; - FIELD_TYPE_DATE = 5; - FIELD_TYPE_DATETIME = 6; - FIELD_TYPE_JSON = 7; - FIELD_TYPE_BLOB = 8; -} - -message Constraint { - ConstraintType type = 1; - repeated string fields = 2; - string expression = 3; -} - -enum ConstraintType { - CONSTRAINT_TYPE_UNSPECIFIED = 0; - CONSTRAINT_TYPE_PRIMARY_KEY = 1; - CONSTRAINT_TYPE_UNIQUE = 2; - CONSTRAINT_TYPE_FOREIGN_KEY = 3; - CONSTRAINT_TYPE_CHECK = 4; - CONSTRAINT_TYPE_FUNCTIONAL_DEPENDENCY = 5; -} - -// ============================================================================= -// Journal Messages -// ============================================================================= - -message JournalRequest { - int64 since = 1; - int32 limit = 2; - string collection = 3; - string actor = 4; -} - -message JournalStreamRequest { - int64 since = 1; - string collection = 2; -} - -message JournalResponse { - repeated JournalEntry entries = 1; - bool has_more = 2; - int64 next_seq = 3; -} - -message JournalEntry { - int64 seq = 1; - google.protobuf.Timestamp timestamp = 2; - OperationType operation = 3; - string collection = 4; - string document_id = 5; - google.protobuf.Struct before = 6; - google.protobuf.Struct after = 7; - Provenance provenance = 8; - string inverse = 9; -} - -enum OperationType { - OPERATION_TYPE_UNSPECIFIED = 0; - OPERATION_TYPE_INSERT = 1; - OPERATION_TYPE_UPDATE = 2; - OPERATION_TYPE_DELETE = 3; - OPERATION_TYPE_CREATE_COLLECTION = 4; - OPERATION_TYPE_DROP_COLLECTION = 5; - OPERATION_TYPE_MIGRATION = 6; -} - -// ============================================================================= -// Normalizer Messages -// ============================================================================= - -message DiscoverRequest { - string collection = 1; - int32 sample_size = 2; - double confidence_threshold = 3; -} - -message DiscoverResponse { - string collection = 1; - repeated FunctionalDependency functional_dependencies = 2; - repeated CandidateKey candidate_keys = 3; -} - -message FunctionalDependency { - repeated string determinant = 1; - string dependent = 2; - double confidence = 3; - ConfidenceTier tier = 4; -} - -enum ConfidenceTier { - CONFIDENCE_TIER_UNSPECIFIED = 0; - CONFIDENCE_TIER_HIGH = 1; - CONFIDENCE_TIER_MEDIUM = 2; - CONFIDENCE_TIER_LOW = 3; -} - -message CandidateKey { - repeated string fields = 1; -} - -message AnalyzeRequest { - string collection = 1; -} - -message AnalyzeResponse { - string collection = 1; - NormalForm current_form = 2; - repeated Violation violations = 3; - repeated Recommendation recommendations = 4; -} - -enum NormalForm { - NORMAL_FORM_UNSPECIFIED = 0; - NORMAL_FORM_1NF = 1; - NORMAL_FORM_2NF = 2; - NORMAL_FORM_3NF = 3; - NORMAL_FORM_BCNF = 4; -} - -message Violation { - ViolationType type = 1; - string description = 2; - repeated string affected_fields = 3; -} - -enum ViolationType { - VIOLATION_TYPE_UNSPECIFIED = 0; - VIOLATION_TYPE_PARTIAL_DEPENDENCY = 1; - VIOLATION_TYPE_TRANSITIVE_DEPENDENCY = 2; - VIOLATION_TYPE_BCNF_VIOLATION = 3; -} - -message Recommendation { - RecommendationAction action = 1; - string description = 2; - NormalForm target_form = 3; - repeated string migration_steps = 4; -} - -enum RecommendationAction { - RECOMMENDATION_ACTION_UNSPECIFIED = 0; - RECOMMENDATION_ACTION_DECOMPOSE = 1; - RECOMMENDATION_ACTION_ADD_CONSTRAINT = 2; - RECOMMENDATION_ACTION_DENORMALIZE = 3; -} - -// ============================================================================= -// Migration Messages -// ============================================================================= - -message MigrationStartRequest { - string collection = 1; - NormalForm target_form = 2; - Provenance provenance = 3; -} - -message MigrationAdvanceRequest { - string migration_id = 1; - Provenance provenance = 2; -} - -message MigrationCommitRequest { - string migration_id = 1; - Provenance provenance = 2; -} - -message MigrationAbortRequest { - string migration_id = 1; - Provenance provenance = 2; -} - -message MigrationStatusRequest { - string migration_id = 1; -} - -message MigrationResponse { - string id = 1; - string collection = 2; - MigrationPhase phase = 3; - google.protobuf.Timestamp started_at = 4; - string narrative = 5; - repeated RewriteRule rewrite_rules = 6; -} - -enum MigrationPhase { - MIGRATION_PHASE_UNSPECIFIED = 0; - MIGRATION_PHASE_ANNOUNCE = 1; - MIGRATION_PHASE_SHADOW = 2; - MIGRATION_PHASE_COMMIT = 3; - MIGRATION_PHASE_ABORTED = 4; - MIGRATION_PHASE_COMPLETE = 5; -} - -message RewriteRule { - string old_path = 1; - string new_path = 2; - string transformation = 3; -} - -// ============================================================================= -// Health Messages -// ============================================================================= - -message HealthRequest {} - -message HealthResponse { - HealthStatus status = 1; - string version = 2; - int64 uptime_seconds = 3; - map checks = 4; -} - -enum HealthStatus { - HEALTH_STATUS_UNSPECIFIED = 0; - HEALTH_STATUS_HEALTHY = 1; - HEALTH_STATUS_DEGRADED = 2; - HEALTH_STATUS_UNHEALTHY = 3; -} - -enum CheckStatus { - CHECK_STATUS_UNSPECIFIED = 0; - CHECK_STATUS_PASS = 1; - CHECK_STATUS_FAIL = 2; -} diff --git a/lithoglyph/api/spec/openapi.yaml b/lithoglyph/api/spec/openapi.yaml deleted file mode 100644 index a33518ed..00000000 --- a/lithoglyph/api/spec/openapi.yaml +++ /dev/null @@ -1,1115 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# Lith REST API - OpenAPI 3.1 Specification - -openapi: 3.1.0 -info: - title: Lith API - description: | - Narrative-first, reversible, audit-grade database API. - - All operations support provenance tracking via the `X-Provenance-Actor` - and `X-Provenance-Rationale` headers. - version: 0.0.4 - license: - name: PMPL-1.0-or-later - url: https://www.gnu.org/licenses/agpl-3.0.html - contact: - name: hyperpolymath - url: https://github.com/hyperpolymath/lith - -servers: - - url: http://localhost:8080/v1 - description: Local development - - url: https://api.lith.example.com/v1 - description: Production - -tags: - - name: Query - description: GQL query execution - - name: Collections - description: Collection management - - name: Journal - description: Audit journal access - - name: Normalizer - description: Self-normalizing operations - - name: Migration - description: Schema migration operations - - name: Health - description: Health and metrics - - name: Bofig Ingest - description: Bofig evidence ingest from Docudactyl Cap'n Proto payloads - -paths: - /query: - post: - tags: [Query] - summary: Execute GQL query - description: | - Execute any GQL statement (SELECT, INSERT, UPDATE, DELETE, etc.) - with optional provenance tracking. - operationId: executeQuery - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/QueryRequest' - responses: - '200': - description: Query executed successfully - content: - application/json: - schema: - $ref: '#/components/schemas/QueryResponse' - '400': - description: Invalid GQL syntax - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - '500': - description: Execution error - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /collections: - get: - tags: [Collections] - summary: List all collections - operationId: listCollections - responses: - '200': - description: List of collections - content: - application/json: - schema: - $ref: '#/components/schemas/CollectionList' - post: - tags: [Collections] - summary: Create a new collection - operationId: createCollection - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateCollectionRequest' - responses: - '201': - description: Collection created - content: - application/json: - schema: - $ref: '#/components/schemas/Collection' - '409': - description: Collection already exists - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /collections/{name}: - get: - tags: [Collections] - summary: Get collection schema - operationId: getCollection - parameters: - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: Collection details - content: - application/json: - schema: - $ref: '#/components/schemas/Collection' - '404': - description: Collection not found - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - delete: - tags: [Collections] - summary: Drop collection - operationId: dropCollection - parameters: - - name: name - in: path - required: true - schema: - type: string - responses: - '204': - description: Collection dropped - '404': - description: Collection not found - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /journal: - get: - tags: [Journal] - summary: Query journal entries - operationId: getJournal - parameters: - - name: since - in: query - description: Sequence number to start from - schema: - type: integer - minimum: 0 - - name: limit - in: query - description: Maximum entries to return - schema: - type: integer - minimum: 1 - maximum: 1000 - default: 100 - - name: collection - in: query - description: Filter by collection name - schema: - type: string - - name: actor - in: query - description: Filter by actor ID - schema: - type: string - responses: - '200': - description: Journal entries - content: - application/json: - schema: - $ref: '#/components/schemas/JournalResponse' - - /normalize/discover: - post: - tags: [Normalizer] - summary: Discover functional dependencies - operationId: discoverDependencies - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/DiscoverRequest' - responses: - '200': - description: Discovered dependencies - content: - application/json: - schema: - $ref: '#/components/schemas/DiscoverResponse' - - /normalize/analyze: - post: - tags: [Normalizer] - summary: Analyze normal form - operationId: analyzeNormalForm - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/AnalyzeRequest' - responses: - '200': - description: Normal form analysis - content: - application/json: - schema: - $ref: '#/components/schemas/AnalyzeResponse' - - /migrate/start: - post: - tags: [Migration] - summary: Start a migration - operationId: startMigration - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/MigrationStartRequest' - responses: - '200': - description: Migration started - content: - application/json: - schema: - $ref: '#/components/schemas/MigrationResponse' - - /migrate/{id}/shadow: - post: - tags: [Migration] - summary: Advance migration to shadow phase - operationId: advanceToShadow - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - '200': - description: Advanced to shadow phase - content: - application/json: - schema: - $ref: '#/components/schemas/MigrationResponse' - - /migrate/{id}/commit: - post: - tags: [Migration] - summary: Commit migration - operationId: commitMigration - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - '200': - description: Migration committed - content: - application/json: - schema: - $ref: '#/components/schemas/MigrationResponse' - - /migrate/{id}/abort: - post: - tags: [Migration] - summary: Abort migration - operationId: abortMigration - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - '200': - description: Migration aborted - content: - application/json: - schema: - $ref: '#/components/schemas/MigrationResponse' - - /bofig/ingest: - post: - tags: [Bofig Ingest] - summary: Ingest a single evidence record - description: | - Ingest a single evidence record into the Bofig collections. - Accepts evidence with optional embedded entities, financial - transactions, and claims. Deduplication by SHA-256 hash. - operationId: ingestEvidence - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/IngestEvidenceRequest' - responses: - '200': - description: Evidence ingested successfully - content: - application/json: - schema: - $ref: '#/components/schemas/IngestResult' - '400': - description: Invalid evidence record - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - '409': - description: Duplicate evidence (SHA-256 hash already exists) - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - '500': - description: Ingest error - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /bofig/batch-ingest: - post: - tags: [Bofig Ingest] - summary: Batch ingest evidence records - description: | - Batch ingest multiple evidence records with shared provenance. - Supports deduplication by SHA-256 hash and continue-on-error - semantics. Returns per-record results with imported/skipped/failed - counts. - operationId: batchIngestEvidence - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/BatchIngestRequest' - responses: - '200': - description: Batch ingest completed - content: - application/json: - schema: - $ref: '#/components/schemas/BatchIngestResult' - '400': - description: Invalid batch request - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - '500': - description: Batch ingest error - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /bofig/ingest/{runId}/status: - get: - tags: [Bofig Ingest] - summary: Get ingest run status - description: | - Get the progress and status of a batch ingest run. - operationId: getIngestStatus - parameters: - - name: runId - in: path - required: true - description: Batch ingest run identifier - schema: - type: string - responses: - '200': - description: Ingest run status - content: - application/json: - schema: - $ref: '#/components/schemas/IngestProgress' - '404': - description: Ingest run not found - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /health: - get: - tags: [Health] - summary: Health check - operationId: healthCheck - responses: - '200': - description: Service is healthy - content: - application/json: - schema: - $ref: '#/components/schemas/HealthResponse' - - /metrics: - get: - tags: [Health] - summary: Prometheus metrics - operationId: getMetrics - responses: - '200': - description: Prometheus metrics - content: - text/plain: - schema: - type: string - -components: - schemas: - QueryRequest: - type: object - required: [gql] - properties: - gql: - type: string - description: GQL statement to execute - example: "SELECT * FROM articles WHERE status = 'published' LIMIT 10" - provenance: - $ref: '#/components/schemas/Provenance' - explain: - type: boolean - description: Return query plan instead of executing - default: false - analyze: - type: boolean - description: Execute and return timing information - default: false - verbose: - type: boolean - description: Include verbose output - default: false - - QueryResponse: - type: object - properties: - rows: - type: array - items: - type: object - additionalProperties: true - rowCount: - type: integer - journalSeq: - type: integer - description: Journal sequence number for this operation - plan: - $ref: '#/components/schemas/QueryPlan' - timing: - $ref: '#/components/schemas/Timing' - - QueryPlan: - type: object - properties: - steps: - type: array - items: - $ref: '#/components/schemas/PlanStep' - estimatedCost: - type: number - rationale: - type: string - - PlanStep: - type: object - properties: - type: - type: string - enum: [scan, filter, project, limit, traverse, insert, update, delete] - collection: - type: string - details: - type: object - additionalProperties: true - - Timing: - type: object - properties: - parseMs: - type: number - planMs: - type: number - executeMs: - type: number - totalMs: - type: number - - Provenance: - type: object - properties: - actor: - type: string - description: Actor identifier (user, system, agent) - example: "editor@news.org" - rationale: - type: string - description: Human-readable reason for the operation - example: "Daily content review" - - Collection: - type: object - properties: - name: - type: string - type: - type: string - enum: [document, edge] - schema: - $ref: '#/components/schemas/Schema' - documentCount: - type: integer - createdAt: - type: string - format: date-time - normalForm: - type: string - enum: [1NF, 2NF, 3NF, BCNF, unknown] - - Schema: - type: object - properties: - fields: - type: array - items: - $ref: '#/components/schemas/Field' - constraints: - type: array - items: - $ref: '#/components/schemas/Constraint' - - Field: - type: object - properties: - name: - type: string - type: - type: string - enum: [string, integer, float, boolean, date, datetime, json, blob] - nullable: - type: boolean - default: true - - Constraint: - type: object - properties: - type: - type: string - enum: [primary_key, unique, foreign_key, check, functional_dependency] - fields: - type: array - items: - type: string - expression: - type: string - - CollectionList: - type: object - properties: - collections: - type: array - items: - $ref: '#/components/schemas/Collection' - total: - type: integer - - CreateCollectionRequest: - type: object - required: [name] - properties: - name: - type: string - type: - type: string - enum: [document, edge] - default: document - schema: - $ref: '#/components/schemas/Schema' - provenance: - $ref: '#/components/schemas/Provenance' - - JournalEntry: - type: object - properties: - seq: - type: integer - description: Sequence number - timestamp: - type: string - format: date-time - operation: - type: string - enum: [insert, update, delete, create_collection, drop_collection, migration] - collection: - type: string - documentId: - type: string - before: - type: object - additionalProperties: true - after: - type: object - additionalProperties: true - provenance: - $ref: '#/components/schemas/Provenance' - inverse: - type: string - description: GQL statement to reverse this operation - - JournalResponse: - type: object - properties: - entries: - type: array - items: - $ref: '#/components/schemas/JournalEntry' - hasMore: - type: boolean - nextSeq: - type: integer - - DiscoverRequest: - type: object - required: [collection] - properties: - collection: - type: string - sampleSize: - type: integer - default: 1000 - confidenceThreshold: - type: number - minimum: 0 - maximum: 1 - default: 0.95 - - DiscoverResponse: - type: object - properties: - collection: - type: string - functionalDependencies: - type: array - items: - $ref: '#/components/schemas/FunctionalDependency' - candidateKeys: - type: array - items: - type: array - items: - type: string - - FunctionalDependency: - type: object - properties: - determinant: - type: array - items: - type: string - dependent: - type: string - confidence: - type: number - tier: - type: string - enum: [high, medium, low] - - AnalyzeRequest: - type: object - required: [collection] - properties: - collection: - type: string - - AnalyzeResponse: - type: object - properties: - collection: - type: string - currentForm: - type: string - enum: [1NF, 2NF, 3NF, BCNF] - violations: - type: array - items: - $ref: '#/components/schemas/Violation' - recommendations: - type: array - items: - $ref: '#/components/schemas/Recommendation' - - Violation: - type: object - properties: - type: - type: string - enum: [partial_dependency, transitive_dependency, bcnf_violation] - description: - type: string - affectedFields: - type: array - items: - type: string - - Recommendation: - type: object - properties: - action: - type: string - enum: [decompose, add_constraint, denormalize] - description: - type: string - targetForm: - type: string - migrationSteps: - type: array - items: - type: string - - MigrationStartRequest: - type: object - required: [collection, targetForm] - properties: - collection: - type: string - targetForm: - type: string - enum: [2NF, 3NF, BCNF] - provenance: - $ref: '#/components/schemas/Provenance' - - MigrationResponse: - type: object - properties: - id: - type: string - collection: - type: string - phase: - type: string - enum: [announce, shadow, commit, aborted, complete] - startedAt: - type: string - format: date-time - narrative: - type: string - - HealthResponse: - type: object - properties: - status: - type: string - enum: [healthy, degraded, unhealthy] - version: - type: string - uptime: - type: integer - description: Uptime in seconds - checks: - type: object - additionalProperties: - type: string - enum: [pass, fail] - - IngestEvidenceRequest: - type: object - required: [investigationId, title, evidenceType, sha256Hash, provenance] - properties: - investigationId: - type: string - description: Investigation this evidence belongs to - title: - type: string - description: Human-readable title for the evidence - evidenceType: - type: string - enum: [court_filing, deposition, testimony, flight_log, financial_record, communication, photograph, video, official_statistics, news_report, document, dataset, interview, affidavit, subpoena, other] - contentText: - type: string - description: Extracted text content (OCR'd if from scanned document) - urlSource: - type: string - description: URL or file path to original source document - sha256Hash: - type: string - description: SHA-256 hash for deduplication and integrity verification - perceptualHash: - type: string - description: Perceptual hash for near-duplicate image detection - ocrConfidence: - type: integer - minimum: 0 - maximum: 100 - description: Tesseract OCR confidence score - language: - type: string - description: ISO 639-1 language code - documentDate: - type: string - format: date-time - description: Date of the original document - entitiesExtracted: - type: array - items: - type: string - description: NER-extracted entity names - keywords: - type: array - items: - type: string - description: Auto-extracted keywords - redactionStatus: - type: string - enum: [clean, redacted, partially_recovered, fully_recovered, not_applicable] - redactionCount: - type: integer - minimum: 0 - extractionRunId: - type: string - description: Docudactyl pipeline run identifier - dublinCoreMetadata: - type: object - additionalProperties: true - description: Dublin Core metadata - sensitivityLevel: - type: string - enum: [public, restricted, sealed, source_protected] - promptScores: - $ref: '#/components/schemas/PromptScoresInput' - notes: - type: string - entities: - type: array - items: - $ref: '#/components/schemas/IngestEntityInput' - financialTransactions: - type: array - items: - $ref: '#/components/schemas/IngestFinancialTransactionInput' - claims: - type: array - items: - $ref: '#/components/schemas/IngestClaimInput' - provenance: - $ref: '#/components/schemas/Provenance' - - PromptScoresInput: - type: object - required: [provenance, replicability, objective, methodology, publication, transparency] - properties: - provenance: - type: integer - minimum: 0 - maximum: 100 - replicability: - type: integer - minimum: 0 - maximum: 100 - objective: - type: integer - minimum: 0 - maximum: 100 - methodology: - type: integer - minimum: 0 - maximum: 100 - publication: - type: integer - minimum: 0 - maximum: 100 - transparency: - type: integer - minimum: 0 - maximum: 100 - - IngestEntityInput: - type: object - required: [primaryName, entityType] - properties: - primaryName: - type: string - entityType: - type: string - enum: [person, organization, location, account, vessel, aircraft] - aliases: - type: array - items: - type: string - description: - type: string - existingEntityId: - type: string - description: Existing entity ID for co-reference linking - matchScore: - type: number - minimum: 0 - maximum: 1 - description: Jaro-Winkler similarity score if matched - - IngestFinancialTransactionInput: - type: object - required: [sourceEntity, destinationEntity, amount, currency, transactionDate] - properties: - sourceEntity: - type: string - description: Entity ID or name of the sender/payer - destinationEntity: - type: string - description: Entity ID or name of the receiver/payee - amount: - type: integer - description: Transaction amount in original currency - currency: - type: string - enum: [USD, GBP, EUR, CHF, JPY, CAD, AUD, OTHER] - transactionDate: - type: string - format: date-time - instrument: - type: string - enum: [wire_transfer, check, cash, credit_card, crypto, trust_payment, shell_company, other] - intermediaryEntity: - type: string - sourceAccount: - type: string - destinationAccount: - type: string - anomalyFlags: - type: array - items: - type: string - enum: [round_number, structuring, unusual_timing, new_counterparty, high_value, cross_border] - notes: - type: string - - IngestClaimInput: - type: object - required: [claimText, claimType, confidence] - properties: - claimText: - type: string - description: The specific factual assertion - claimType: - type: string - enum: [factual, temporal, financial, testimonial, documentary, circumstantial, corroborative, contradictory] - confidence: - type: integer - minimum: 0 - maximum: 100 - speakerId: - type: string - claimDate: - type: string - format: date-time - assertionDate: - type: string - format: date-time - promptScores: - $ref: '#/components/schemas/PromptScoresInput' - - BatchIngestRequest: - type: object - required: [investigationId, extractionRunId, records, provenance] - properties: - investigationId: - type: string - description: Investigation this batch belongs to - extractionRunId: - type: string - description: Docudactyl extraction run ID for the batch - records: - type: array - items: - $ref: '#/components/schemas/IngestEvidenceRequest' - provenance: - $ref: '#/components/schemas/Provenance' - deduplicateByHash: - type: boolean - default: true - description: Skip records that already exist by SHA-256 hash - continueOnError: - type: boolean - default: true - description: Continue on individual record failure - - IngestResult: - type: object - properties: - success: - type: boolean - evidenceId: - type: string - journalSeq: - type: integer - entitiesProcessed: - type: integer - transactionsProcessed: - type: integer - claimsProcessed: - type: integer - errors: - type: array - items: - $ref: '#/components/schemas/IngestError' - - BatchIngestResult: - type: object - properties: - runId: - type: string - totalSubmitted: - type: integer - importedCount: - type: integer - skippedCount: - type: integer - failedCount: - type: integer - results: - type: array - items: - $ref: '#/components/schemas/IngestResult' - timing: - $ref: '#/components/schemas/Timing' - - IngestProgress: - type: object - properties: - runId: - type: string - phase: - type: string - enum: [validating, deduplicating, inserting_evidence, resolving_entities, inserting_transactions, inserting_claims, creating_relationships, complete, failed] - progress: - type: number - minimum: 0 - maximum: 1 - description: Progress as a fraction (0.0 to 1.0) - processedCount: - type: integer - totalCount: - type: integer - message: - type: string - currentRecord: - type: string - - IngestError: - type: object - properties: - recordIndex: - type: integer - description: Zero-based index of the failed record - code: - type: string - message: - type: string - sha256Hash: - type: string - - Error: - type: object - properties: - code: - type: string - message: - type: string - details: - type: object - additionalProperties: true - - securitySchemes: - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - apiKey: - type: apiKey - in: header - name: X-API-Key - -security: - - bearerAuth: [] - - apiKey: [] diff --git a/lithoglyph/api/src/auth.zig b/lithoglyph/api/src/auth.zig deleted file mode 100644 index 5007bd1e..00000000 --- a/lithoglyph/api/src/auth.zig +++ /dev/null @@ -1,274 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Lithoglyph API Server - Authentication -// -// Authentication tokens are loaded from environment variables at startup. -// NEVER hardcode tokens, secrets, or API keys in source code. - -const std = @import("std"); -const config = @import("config.zig"); - -const log = std.log.scoped(.auth); - -/// Environment variable name for the auth token used in development/testing. -/// In production, use LITH_JWT_SECRET (read via config) for real JWT validation. -const AUTH_TOKEN_ENV = "LITHOGLYPH_AUTH_TOKEN"; - -/// Error set for authentication operations -pub const AuthError = error{ - /// LITHOGLYPH_AUTH_TOKEN environment variable is not set - AuthTokenNotConfigured, - /// Token provided does not match the configured token - InvalidToken, - /// Token has expired or is malformed - MalformedToken, - /// JWT secret not configured (LITH_JWT_SECRET missing) - JwtSecretNotConfigured, -}; - -var allocator: std.mem.Allocator = undefined; -var cfg: *const config.Config = undefined; - -// API key storage (in production, use persistent storage) -var api_keys: std.StringHashMap(ApiKey) = undefined; - -pub const ApiKey = struct { - name: []const u8, - scopes: []const Scope, - created_at: i64, - expires_at: ?i64, -}; - -pub const Scope = enum { - read, - write, - admin, - migrate, -}; - -pub fn init(alloc: std.mem.Allocator, config_ptr: *const config.Config) !void { - allocator = alloc; - cfg = config_ptr; - api_keys = std.StringHashMap(ApiKey).init(alloc); - - // Load API key from environment variable — NEVER hardcode keys in source - if (std.posix.getenv(AUTH_TOKEN_ENV)) |env_key| { - if (env_key.len > 0) { - try api_keys.put(env_key, .{ - .name = "Environment Key", - .scopes = &[_]Scope{ .read, .write, .admin, .migrate }, - .created_at = std.time.timestamp(), - .expires_at = null, - }); - log.info("Authentication initialized with environment key", .{}); - } else { - log.warn("{s} is set but empty — no API key registered", .{AUTH_TOKEN_ENV}); - } - } else { - log.warn("{s} not set — API key authentication disabled. " ++ - "Set this environment variable to enable token-based auth.", .{AUTH_TOKEN_ENV}); - } -} - -pub fn deinit() void { - api_keys.deinit(); -} - -pub fn validateRequest(request: *std.http.Server.Request) !bool { - // Check for API key - if (getHeader(request, cfg.api_key_header)) |key| { - return validateApiKey(key); - } - - // Check for Bearer token - if (getHeader(request, "authorization")) |auth| { - if (std.mem.startsWith(u8, auth, "Bearer ")) { - const token = auth["Bearer ".len..]; - return validateJWT(token); - } - } - - return false; -} - -pub fn validateApiKey(key: []const u8) bool { - if (api_keys.get(key)) |_| { - return true; - } - return false; -} - -pub fn validateJWT(token: []const u8) bool { - // TODO: Replace with real JWT validation using HMAC-SHA256 or RS256. - // This stub only checks structural validity (3 dot-separated parts) - // and verifies the JWT secret is configured. It does NOT verify - // the signature, expiration, or claims. A proper JWT library - // (e.g. zig-jwt or a C binding to libjwt) is required for - // production deployment. - - // Reject if no JWT secret is configured - if (cfg.jwt_secret == null) { - log.warn("JWT validation failed: LITH_JWT_SECRET not set", .{}); - return false; - } - - // JWT format: header.payload.signature (exactly 3 parts) - var parts = std.mem.splitScalar(u8, token, '.'); - - const header = parts.next() orelse return false; - const payload = parts.next() orelse return false; - const signature = parts.next() orelse return false; - - // Verify there are exactly 3 parts - if (parts.next() != null) return false; - - // Reject empty segments - if (header.len == 0 or payload.len == 0 or signature.len == 0) return false; - - // TODO: Verify signature using HMAC-SHA256 with cfg.jwt_secret.? - // TODO: Decode payload, check "exp" claim against current time - // TODO: Validate "iss", "aud" claims against expected values - - log.warn("JWT accepted without signature verification — NOT SAFE FOR PRODUCTION", .{}); - return true; -} - -pub fn getScopes(request: *std.http.Server.Request) []const Scope { - if (getHeader(request, cfg.api_key_header)) |key| { - if (api_keys.get(key)) |api_key| { - return api_key.scopes; - } - } - return &[_]Scope{}; -} - -pub fn hasScope(request: *std.http.Server.Request, required: Scope) bool { - const scopes = getScopes(request); - for (scopes) |s| { - if (s == required or s == .admin) { - return true; - } - } - return false; -} - -fn getHeader(request: *std.http.Server.Request, name: []const u8) ?[]const u8 { - var iter = request.iterateHeaders(); - while (iter.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, name)) { - return header.value; - } - } - return null; -} - -// ============================================================================= -// JWT Helpers -// ============================================================================= - -pub const Claims = struct { - sub: []const u8, // Subject (user ID) - exp: i64, // Expiration time - iat: i64, // Issued at - scopes: []const Scope, -}; - -pub fn createJWT(claims: Claims) AuthError![]const u8 { - _ = claims; - // TODO: Implement real JWT creation with HMAC-SHA256 signature. - // Steps needed: - // 1. Base64url-encode JSON header {"alg":"HS256","typ":"JWT"} - // 2. Base64url-encode JSON payload from claims - // 3. Sign header.payload with LITH_JWT_SECRET using HMAC-SHA256 - // 4. Return header.payload.signature - // - // Until implemented, callers should use API key auth instead. - - const secret = std.posix.getenv("LITH_JWT_SECRET") orelse { - log.err("Cannot create JWT: LITH_JWT_SECRET environment variable not set", .{}); - return AuthError.JwtSecretNotConfigured; - }; - _ = secret; - - // Return error rather than a fake token — callers must handle this - return AuthError.JwtSecretNotConfigured; -} - -pub fn parseJWT(token: []const u8) AuthError!Claims { - // TODO: Implement real JWT parsing and signature verification. - // Steps needed: - // 1. Split token on '.' - // 2. Base64url-decode header and payload - // 3. Verify signature using LITH_JWT_SECRET - // 4. Parse claims from payload JSON - // 5. Validate exp > now, check iss/aud - // - // Until implemented, reject all tokens. - - _ = token; - - const secret = std.posix.getenv("LITH_JWT_SECRET") orelse { - log.err("Cannot parse JWT: LITH_JWT_SECRET environment variable not set", .{}); - return AuthError.JwtSecretNotConfigured; - }; - _ = secret; - - // For now, return a minimal read-only claim set with short expiry. - // This is a stub — real implementation must verify the signature first. - return .{ - .sub = "unverified", - .exp = std.time.timestamp() + 300, // 5 minutes, not 1 hour - .iat = std.time.timestamp(), - .scopes = &[_]Scope{.read}, // Read-only until real verification - }; -} - -// ============================================================================= -// mTLS Support (for gRPC) -// ============================================================================= - -pub const TLSConfig = struct { - cert_path: []const u8, - key_path: []const u8, - ca_path: ?[]const u8 = null, // For mTLS - require_client_cert: bool = false, -}; - -pub fn validateClientCert(cert_chain: []const u8) bool { - // TODO: Validate client certificate chain against configured CA. - // This is a security-critical stub that must be implemented - // before enabling mTLS in production. - _ = cert_chain; - log.warn("Client certificate validation not implemented — accepting all certs", .{}); - return true; -} - -test "api key from environment" { - // With no environment variable set, no keys should be registered - // (we cannot set env vars in Zig tests, so we test the empty case) - const alloc = std.testing.allocator; - var keys = std.StringHashMap(ApiKey).init(alloc); - defer keys.deinit(); - - // Verify that unknown keys are rejected - try std.testing.expect(keys.get("nonexistent-key") == null); -} - -test "jwt structure validation" { - const valid_jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signature"; - var parts = std.mem.splitScalar(u8, valid_jwt, '.'); - - try std.testing.expect(parts.next() != null); - try std.testing.expect(parts.next() != null); - try std.testing.expect(parts.next() != null); - try std.testing.expect(parts.next() == null); -} - -test "empty jwt segments rejected" { - // A JWT with empty segments should be structurally invalid - const bad_jwt = ".payload.signature"; - var parts = std.mem.splitScalar(u8, bad_jwt, '.'); - const header = parts.next() orelse unreachable; - try std.testing.expect(header.len == 0); // Would be rejected by validateJWT -} diff --git a/lithoglyph/api/src/bridge_client.zig b/lithoglyph/api/src/bridge_client.zig deleted file mode 100644 index 52fa8f8d..00000000 --- a/lithoglyph/api/src/bridge_client.zig +++ /dev/null @@ -1,584 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Bridge Client -// -// Wraps Form.Bridge FFI calls for the API server, converting between -// JSON (HTTP) and CBOR (bridge) formats. - -const std = @import("std"); -const config = @import("config.zig"); - -const log = std.log.scoped(.bridge_client); - -// FFI type definitions (matching bridge.zig) -pub const LithBlob = extern struct { - ptr: ?[*]const u8, - len: usize, - - pub fn empty() LithBlob { - return .{ .ptr = null, .len = 0 }; - } - - pub fn fromSlice(data: []const u8) LithBlob { - return .{ .ptr = data.ptr, .len = data.len }; - } - - pub fn toSlice(self: LithBlob) ?[]const u8 { - if (self.ptr) |p| { - return p[0..self.len]; - } - return null; - } -}; - -pub const LithStatus = enum(i32) { - ok = 0, - err_invalid_argument = -1, - err_not_found = -2, - err_io_error = -3, - err_out_of_memory = -4, - err_internal = -5, - err_txn_not_active = -6, - err_txn_already_committed = -7, - err_constraint_violation = -8, - err_not_implemented = -100, -}; - -pub const LithResult = extern struct { - result: LithBlob, - provenance: LithBlob, - status: LithStatus, - error_blob: LithBlob, -}; - -pub const LithTxnMode = enum(u8) { - read_only = 0, - read_write = 1, -}; - -pub const LithRenderOpts = extern struct { - include_provenance: bool = true, - canonical: bool = true, - pretty: bool = false, -}; - -// Opaque handles -pub const LithDb = opaque {}; -pub const LithTxn = opaque {}; - -// External bridge functions (linked from core-zig) -extern fn lith_db_open( - path_ptr: [*]const u8, - path_len: usize, - opts_ptr: ?[*]const u8, - opts_len: usize, - out_db: *?*LithDb, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_db_close(db: ?*LithDb) LithStatus; - -extern fn lith_txn_begin( - db: ?*LithDb, - mode: LithTxnMode, - out_txn: *?*LithTxn, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_txn_commit(txn: ?*LithTxn, out_err: *LithBlob) LithStatus; -extern fn lith_txn_abort(txn: ?*LithTxn) LithStatus; - -extern fn lith_apply( - txn: ?*LithTxn, - op_ptr: [*]const u8, - op_len: usize, -) LithResult; - -extern fn lith_introspect_schema( - db: ?*LithDb, - out_schema: *LithBlob, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_introspect_constraints( - db: ?*LithDb, - out_constraints: *LithBlob, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_render_journal( - db: ?*LithDb, - since: u64, - opts: LithRenderOpts, - out_text: *LithBlob, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_render_block( - db: ?*LithDb, - block_id: u64, - opts: LithRenderOpts, - out_text: *LithBlob, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_proof_verify( - proof_ptr: [*]const u8, - proof_len: usize, - out_valid: *bool, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_proof_init_builtins() LithStatus; -extern fn lith_blob_free(blob: *LithBlob) void; -extern fn lith_version() u32; - -// ============================================================================= -// Bridge Client -// ============================================================================= - -var allocator: std.mem.Allocator = undefined; -var db_handle: ?*LithDb = null; -var is_initialized: bool = false; - -pub fn init(alloc: std.mem.Allocator, cfg: *const config.Config) !void { - allocator = alloc; - - // Initialize built-in proof verifiers - const verifier_status = lith_proof_init_builtins(); - if (verifier_status != .ok) { - log.warn("Failed to initialize proof verifiers: {}", .{verifier_status}); - } - - // Open database - var err_blob: LithBlob = LithBlob.empty(); - const status = lith_db_open( - cfg.db_path.ptr, - cfg.db_path.len, - null, - 0, - &db_handle, - &err_blob, - ); - - if (status != .ok) { - if (err_blob.toSlice()) |err_data| { - log.err("Failed to open database: {s}", .{err_data}); - lith_blob_free(&err_blob); - } - return error.DatabaseOpenFailed; - } - - is_initialized = true; - log.info("Bridge client initialized, database: {s}", .{cfg.db_path}); -} - -pub fn deinit() void { - if (db_handle) |db| { - _ = lith_db_close(db); - db_handle = null; - } - is_initialized = false; -} - -pub fn isInitialized() bool { - return is_initialized; -} - -// ============================================================================= -// Query Execution -// ============================================================================= - -pub const QueryResult = struct { - data: []const u8, - provenance: ?[]const u8, - rows_affected: u64, - - pub fn deinit(self: *QueryResult, alloc: std.mem.Allocator) void { - alloc.free(self.data); - if (self.provenance) |p| { - alloc.free(p); - } - } -}; - -pub fn executeQuery(gql: []const u8, provenance: ?QueryProvenance) !QueryResult { - if (!is_initialized) { - return error.NotInitialized; - } - - // Begin transaction - var txn: ?*LithTxn = null; - var txn_err: LithBlob = LithBlob.empty(); - - const txn_status = lith_txn_begin(db_handle, .read_write, &txn, &txn_err); - if (txn_status != .ok) { - if (txn_err.toSlice()) |err| { - log.err("Transaction begin failed: {s}", .{err}); - lith_blob_free(&txn_err); - } - return error.TransactionFailed; - } - defer { - if (txn != null) { - _ = lith_txn_abort(txn); - } - } - - // Encode operation as CBOR (simplified - just wrap GQL string) - var op_buffer: [4096]u8 = undefined; - const op_len = encodeLithqlOperation(&op_buffer, gql, provenance) catch { - return error.EncodingFailed; - }; - - // Execute operation - const result = lith_apply(txn, &op_buffer, op_len); - - if (result.status != .ok) { - if (result.error_blob.toSlice()) |err| { - log.err("Apply failed: {s}", .{err}); - } - return error.ApplyFailed; - } - - // Commit transaction - var commit_err: LithBlob = LithBlob.empty(); - const commit_status = lith_txn_commit(txn, &commit_err); - txn = null; // Mark as consumed - - if (commit_status != .ok) { - if (commit_err.toSlice()) |err| { - log.err("Commit failed: {s}", .{err}); - lith_blob_free(&commit_err); - } - return error.CommitFailed; - } - - // Convert result to JSON - const json_data = try cborToJson(allocator, result.result.toSlice() orelse &[_]u8{}); - var prov_json: ?[]const u8 = null; - if (result.provenance.toSlice()) |prov_cbor| { - prov_json = cborToJson(allocator, prov_cbor) catch null; - } - - return QueryResult{ - .data = json_data, - .provenance = prov_json, - .rows_affected = 1, // Placeholder - }; -} - -pub const QueryProvenance = struct { - actor: ?[]const u8 = null, - rationale: ?[]const u8 = null, -}; - -// ============================================================================= -// Schema Operations -// ============================================================================= - -pub const CollectionInfo = struct { - name: []const u8, - schema_version: u32, - document_count: u64, -}; - -pub fn listCollections() ![]CollectionInfo { - if (!is_initialized) { - return error.NotInitialized; - } - - var schema_blob: LithBlob = LithBlob.empty(); - var err_blob: LithBlob = LithBlob.empty(); - - const status = lith_introspect_schema(db_handle, &schema_blob, &err_blob); - if (status != .ok) { - if (err_blob.toSlice()) |err| { - log.err("Schema introspection failed: {s}", .{err}); - lith_blob_free(&err_blob); - } - return error.IntrospectionFailed; - } - defer lith_blob_free(&schema_blob); - - // Parse CBOR schema response (placeholder - return empty list) - const collections = try allocator.alloc(CollectionInfo, 0); - return collections; -} - -pub fn getCollection(name: []const u8) !?CollectionInfo { - const collections = try listCollections(); - defer allocator.free(collections); - - for (collections) |col| { - if (std.mem.eql(u8, col.name, name)) { - return col; - } - } - return null; -} - -pub fn createCollection(name: []const u8, schema_json: []const u8) !void { - _ = name; - _ = schema_json; - // Create collection via GQL CREATE COLLECTION statement - // For now, placeholder - return error.NotImplemented; -} - -pub fn dropCollection(name: []const u8) !void { - _ = name; - // Drop collection via GQL DROP COLLECTION statement - // For now, placeholder - return error.NotImplemented; -} - -// ============================================================================= -// Journal Operations -// ============================================================================= - -pub const JournalEntry = struct { - sequence: u64, - timestamp: []const u8, - operation: []const u8, - collection: ?[]const u8, - actor: ?[]const u8, -}; - -pub fn getJournal(since: u64, limit: u32) ![]JournalEntry { - if (!is_initialized) { - return error.NotInitialized; - } - - var journal_blob: LithBlob = LithBlob.empty(); - var err_blob: LithBlob = LithBlob.empty(); - - const opts = LithRenderOpts{ - .include_provenance = true, - .canonical = true, - .pretty = false, - }; - - const status = lith_render_journal(db_handle, since, opts, &journal_blob, &err_blob); - if (status != .ok) { - if (err_blob.toSlice()) |err| { - log.err("Journal render failed: {s}", .{err}); - lith_blob_free(&err_blob); - } - return error.JournalRenderFailed; - } - defer lith_blob_free(&journal_blob); - - // Parse CBOR journal response (placeholder - return empty list) - _ = limit; - const entries = try allocator.alloc(JournalEntry, 0); - return entries; -} - -// ============================================================================= -// Normalization Operations -// ============================================================================= - -pub const FunctionalDependency = struct { - determinant: []const []const u8, - dependent: []const u8, - confidence: f32, -}; - -pub const NormalFormAnalysis = struct { - current_form: []const u8, - violations: []const []const u8, - suggestions: []const []const u8, -}; - -pub fn discoverDependencies(collection: []const u8, sample_size: u32) ![]FunctionalDependency { - _ = collection; - _ = sample_size; - // Calls Form.Normalizer via bridge - return error.NotImplemented; -} - -pub fn analyzeNormalForm(collection: []const u8) !NormalFormAnalysis { - _ = collection; - return error.NotImplemented; -} - -// ============================================================================= -// Migration Operations -// ============================================================================= - -pub const MigrationState = enum { - announced, - shadow_running, - shadow_complete, - committed, - aborted, -}; - -pub const Migration = struct { - id: []const u8, - state: MigrationState, - source_collection: []const u8, - target_schema: []const u8, - created_at: []const u8, -}; - -pub fn startMigration(source: []const u8, target_schema: []const u8) !Migration { - _ = source; - _ = target_schema; - return error.NotImplemented; -} - -pub fn getMigration(id: []const u8) !?Migration { - _ = id; - return error.NotImplemented; -} - -pub fn advanceMigration(id: []const u8, action: MigrationAction) !void { - _ = id; - _ = action; - return error.NotImplemented; -} - -pub const MigrationAction = enum { - start_shadow, - commit, - abort, -}; - -// ============================================================================= -// Health Check -// ============================================================================= - -pub const HealthStatus = struct { - status: []const u8, - version: []const u8, - uptime_seconds: u64, - journal_head: u64, - collections_count: u32, -}; - -pub fn getHealth() HealthStatus { - const version_num = lith_version(); - const major = version_num / 10000; - const minor = (version_num % 10000) / 100; - const patch = version_num % 100; - - var version_buf: [32]u8 = undefined; - const version_str = std.fmt.bufPrint(&version_buf, "{}.{}.{}", .{ major, minor, patch }) catch "0.0.0"; - - return HealthStatus{ - .status = if (is_initialized) "healthy" else "degraded", - .version = version_str, - .uptime_seconds = 0, // Would need to track start time - .journal_head = 0, // Would need to query from db - .collections_count = 0, - }; -} - -// ============================================================================= -// CBOR Encoding/Decoding Helpers -// ============================================================================= - -fn encodeLithqlOperation(buffer: []u8, gql: []const u8, prov: ?QueryProvenance) !usize { - // Simplified CBOR encoding for GQL operation - // In production, would use proper CBOR encoder - - // CBOR map with 2-3 entries - var offset: usize = 0; - - // Map header (0xa2 = map of 2 items, 0xa3 = map of 3 items) - const has_prov = prov != null and (prov.?.actor != null or prov.?.rationale != null); - buffer[offset] = if (has_prov) 0xa3 else 0xa2; - offset += 1; - - // Key: "op" - buffer[offset] = 0x62; // text of 2 bytes - offset += 1; - buffer[offset] = 'o'; - offset += 1; - buffer[offset] = 'p'; - offset += 1; - - // Value: "query" - buffer[offset] = 0x65; // text of 5 bytes - offset += 1; - @memcpy(buffer[offset .. offset + 5], "query"); - offset += 5; - - // Key: "gql" - buffer[offset] = 0x64; // text of 4 bytes - offset += 1; - @memcpy(buffer[offset .. offset + 4], "gql"); - offset += 4; - - // Value: gql string - if (gql.len < 24) { - buffer[offset] = @as(u8, 0x60) + @as(u8, @intCast(gql.len)); - offset += 1; - } else if (gql.len < 256) { - buffer[offset] = 0x78; // text with 1-byte length - offset += 1; - buffer[offset] = @intCast(gql.len); - offset += 1; - } else { - buffer[offset] = 0x79; // text with 2-byte length - offset += 1; - buffer[offset] = @intCast(gql.len >> 8); - offset += 1; - buffer[offset] = @intCast(gql.len & 0xFF); - offset += 1; - } - @memcpy(buffer[offset .. offset + gql.len], gql); - offset += gql.len; - - // Optional provenance - if (has_prov) { - // Key: "prov" - buffer[offset] = 0x64; - offset += 1; - @memcpy(buffer[offset .. offset + 4], "prov"); - offset += 4; - - // Value: map with actor/rationale (simplified) - buffer[offset] = 0xa0; // empty map for now - offset += 1; - } - - return offset; -} - -fn cborToJson(alloc: std.mem.Allocator, cbor_data: []const u8) ![]const u8 { - // Simplified CBOR to JSON conversion - // In production, would use proper CBOR decoder - - if (cbor_data.len == 0) { - return try alloc.dupe(u8, "{}"); - } - - // For PoC, return placeholder JSON based on first byte - const first = cbor_data[0]; - - if (first >= 0xa0 and first <= 0xbf) { - // Map - return as object - return try alloc.dupe(u8, "{\"status\":\"ok\"}"); - } else if (first >= 0x80 and first <= 0x9f) { - // Array - return try alloc.dupe(u8, "[]"); - } else { - return try alloc.dupe(u8, "{}"); - } -} - -// ============================================================================= -// Tests -// ============================================================================= - -test "health check without init" { - const health = getHealth(); - try std.testing.expectEqualStrings("degraded", health.status); -} - -test "query without init fails" { - const result = executeQuery("SELECT * FROM test", null); - try std.testing.expectError(error.NotInitialized, result); -} diff --git a/lithoglyph/api/src/config.zig b/lithoglyph/api/src/config.zig deleted file mode 100644 index f8ab0403..00000000 --- a/lithoglyph/api/src/config.zig +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Configuration - -const std = @import("std"); - -pub const Config = struct { - allocator: std.mem.Allocator, - - // Server settings - host: []const u8, - port: u16, - version: []const u8, - - // Authentication - jwt_secret: ?[]const u8, - api_key_header: []const u8, - require_auth: bool, - - // Database connection - db_path: []const u8, - - // Logging - log_level: std.log.Level, - - // Limits - max_request_size: usize, - max_response_rows: usize, - request_timeout_ms: u64, - - // TLS (optional) - tls_cert_path: ?[]const u8, - tls_key_path: ?[]const u8, - - pub fn deinit(self: *const Config) void { - // Free allocated strings if needed - _ = self; - } -}; - -pub fn load(allocator: std.mem.Allocator) !*const Config { - const config = try allocator.create(Config); - - config.* = Config{ - .allocator = allocator, - - // Server defaults - .host = getEnvOrDefault("LITH_HOST", "127.0.0.1"), - .port = getEnvPort("LITH_PORT", 8080), - .version = "0.0.4", - - // Auth defaults - .jwt_secret = std.posix.getenv("LITH_JWT_SECRET"), - .api_key_header = getEnvOrDefault("LITH_API_KEY_HEADER", "X-API-Key"), - .require_auth = getEnvBool("LITH_REQUIRE_AUTH", false), - - // Database - .db_path = getEnvOrDefault("LITH_DB_PATH", "./lithoglyph.dat"), - - // Logging - .log_level = .info, - - // Limits - .max_request_size = 10 * 1024 * 1024, // 10 MB - .max_response_rows = 10000, - .request_timeout_ms = 30000, // 30 seconds - - // TLS - .tls_cert_path = std.posix.getenv("LITH_TLS_CERT"), - .tls_key_path = std.posix.getenv("LITH_TLS_KEY"), - }; - - return config; -} - -fn getEnvOrDefault(key: []const u8, default: []const u8) []const u8 { - return std.posix.getenv(key) orelse default; -} - -fn getEnvPort(key: []const u8, default: u16) u16 { - const env = std.posix.getenv(key) orelse return default; - return std.fmt.parseInt(u16, env, 10) catch default; -} - -fn getEnvBool(key: []const u8, default: bool) bool { - const env = std.posix.getenv(key) orelse return default; - return std.mem.eql(u8, env, "true") or std.mem.eql(u8, env, "1"); -} - -test "config loading" { - const allocator = std.testing.allocator; - const cfg = try load(allocator); - defer allocator.destroy(cfg); - - try std.testing.expectEqualStrings("127.0.0.1", cfg.host); - try std.testing.expectEqual(@as(u16, 8080), cfg.port); -} diff --git a/lithoglyph/api/src/graphql.zig b/lithoglyph/api/src/graphql.zig deleted file mode 100644 index 7bd782a9..00000000 --- a/lithoglyph/api/src/graphql.zig +++ /dev/null @@ -1,318 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - GraphQL Handler -// -// GraphQL endpoint for Lithoglyph operations -// Supports queries, mutations, and subscriptions - -const std = @import("std"); -const json = std.json; -const config = @import("config.zig"); -const bridge = @import("bridge_client.zig"); -const websocket = @import("websocket.zig"); - -const log = std.log.scoped(.graphql); - -pub fn handleRequest( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - cfg: *const config.Config, -) !void { - const method = request.head.method; - - // Check for WebSocket upgrade (for subscriptions) - if (websocket.isUpgradeRequest(request)) { - try websocket.handleUpgrade(allocator, request, cfg); - return; - } - - switch (method) { - .GET => try handleGraphiQL(request), - .POST => try handleGraphQLQuery(allocator, request), - .OPTIONS => try handleCORS(request), - else => try sendMethodNotAllowed(request), - } -} - -fn handleGraphiQL(request: *std.http.Server.Request) !void { - // Return GraphiQL HTML interface - const html = - \\ - \\ - \\ - \\ Lithoglyph GraphQL - \\ - \\ - \\ - \\ - \\
- \\ - \\ - \\ - \\ - \\ - \\ - ; - - request.respond(html, .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "text/html; charset=utf-8" }, - }, - }) catch {}; -} - -fn handleGraphQLQuery(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - // Read request body - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 10 * 1024 * 1024); - defer allocator.free(body); - - // Parse GraphQL request - const parsed = json.parseFromSlice(GraphQLRequest, allocator, body, .{}) catch { - try sendError(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - - log.info("GraphQL query: {s}", .{req.query[0..@min(100, req.query.len)]}); - - // Parse and execute query - const result = try executeQuery(allocator, req); - defer allocator.free(result); - - request.respond(result, .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -const GraphQLRequest = struct { - query: []const u8, - operationName: ?[]const u8 = null, - variables: ?json.Value = null, -}; - -fn executeQuery(allocator: std.mem.Allocator, req: GraphQLRequest) ![]const u8 { - // Simple query parsing - look for operation type - const query = req.query; - - // Check for introspection query - if (std.mem.indexOf(u8, query, "__schema") != null or - std.mem.indexOf(u8, query, "__type") != null) - { - return try executeIntrospection(allocator); - } - - // Check operation type - if (std.mem.indexOf(u8, query, "mutation")) |_| { - return try executeMutation(allocator, query); - } else if (std.mem.indexOf(u8, query, "subscription")) |_| { - return try allocator.dupe(u8, - \\{"errors":[{"message":"Subscriptions not supported over HTTP. Use WebSocket."}]} - ); - } else { - return try executeQueryOperation(allocator, query); - } -} - -fn executeQueryOperation(allocator: std.mem.Allocator, query: []const u8) ![]const u8 { - // Route to resolver based on field - if (std.mem.indexOf(u8, query, "collections") != null) { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "collections": { - \\ "edges": [ - \\ {"node": {"name": "articles", "type": "DOCUMENT", "documentCount": 1234}}, - \\ {"node": {"name": "users", "type": "DOCUMENT", "documentCount": 567}} - \\ ], - \\ "totalCount": 2 - \\ } - \\ } - \\} - ); - } else if (std.mem.indexOf(u8, query, "journal") != null) { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "journal": { - \\ "edges": [ - \\ { - \\ "node": { - \\ "seq": 42, - \\ "operation": "INSERT", - \\ "collection": "articles" - \\ } - \\ } - \\ ], - \\ "hasMore": false - \\ } - \\ } - \\} - ); - } else if (std.mem.indexOf(u8, query, "health") != null) { - const health = bridge.getHealth(); - var response_buffer: std.ArrayList(u8) = .empty; - errdefer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print( - \\{{"data":{{"health":{{"status":"{s}","version":"{s}","uptimeSeconds":{d}}}}}}} - , .{ - if (std.mem.eql(u8, health.status, "healthy")) "HEALTHY" else "DEGRADED", - health.version, - health.uptime_seconds, - }); - - return try response_buffer.toOwnedSlice(allocator); - } else if (std.mem.indexOf(u8, query, "query(") != null or - std.mem.indexOf(u8, query, "query (") != null) - { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "query": { - \\ "rows": [], - \\ "rowCount": 0, - \\ "journalSeq": 42 - \\ } - \\ } - \\} - ); - } else { - return try allocator.dupe(u8, - \\{"errors":[{"message":"Unknown query field"}]} - ); - } -} - -fn executeMutation(allocator: std.mem.Allocator, query: []const u8) ![]const u8 { - if (std.mem.indexOf(u8, query, "createCollection") != null) { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "createCollection": { - \\ "name": "new_collection", - \\ "type": "DOCUMENT", - \\ "documentCount": 0 - \\ } - \\ } - \\} - ); - } else if (std.mem.indexOf(u8, query, "execute") != null) { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "execute": { - \\ "affectedCount": 1, - \\ "journalSeq": 43 - \\ } - \\ } - \\} - ); - } else if (std.mem.indexOf(u8, query, "startMigration") != null) { - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "startMigration": { - \\ "id": "mig-001", - \\ "phase": "ANNOUNCE", - \\ "narrative": "Migration announced" - \\ } - \\ } - \\} - ); - } else { - return try allocator.dupe(u8, - \\{"errors":[{"message":"Unknown mutation field"}]} - ); - } -} - -fn executeIntrospection(allocator: std.mem.Allocator) ![]const u8 { - // Return simplified introspection result - return try allocator.dupe(u8, - \\{ - \\ "data": { - \\ "__schema": { - \\ "types": [ - \\ {"name": "Query", "kind": "OBJECT"}, - \\ {"name": "Mutation", "kind": "OBJECT"}, - \\ {"name": "Subscription", "kind": "OBJECT"}, - \\ {"name": "Collection", "kind": "OBJECT"}, - \\ {"name": "JournalEntry", "kind": "OBJECT"}, - \\ {"name": "QueryResult", "kind": "OBJECT"}, - \\ {"name": "Migration", "kind": "OBJECT"} - \\ ], - \\ "queryType": {"name": "Query"}, - \\ "mutationType": {"name": "Mutation"}, - \\ "subscriptionType": {"name": "Subscription"} - \\ } - \\ } - \\} - ); -} - -fn handleCORS(request: *std.http.Server.Request) !void { - request.respond("", .{ - .status = .no_content, - .extra_headers = &.{ - .{ .name = "access-control-allow-origin", .value = "*" }, - .{ .name = "access-control-allow-methods", .value = "GET, POST, OPTIONS" }, - .{ .name = "access-control-allow-headers", .value = "content-type, authorization" }, - }, - }) catch {}; -} - -fn sendError(request: *std.http.Server.Request, message: []const u8) !void { - _ = message; - const body = - \\{"errors":[{"message":"Invalid request"}]} - ; - - request.respond(body, .{ - .status = .bad_request, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -fn sendMethodNotAllowed(request: *std.http.Server.Request) !void { - const body = - \\{"errors":[{"message":"Method not allowed"}]} - ; - - request.respond(body, .{ - .status = .method_not_allowed, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -test "graphql request parsing" { - const allocator = std.testing.allocator; - - const body = - \\{"query": "{ collections { edges { node { name } } } }"} - ; - - const parsed = try json.parseFromSlice(GraphQLRequest, allocator, body, .{}); - defer parsed.deinit(); - - try std.testing.expect(std.mem.indexOf(u8, parsed.value.query, "collections") != null); -} diff --git a/lithoglyph/api/src/grpc.zig b/lithoglyph/api/src/grpc.zig deleted file mode 100644 index 076ff065..00000000 --- a/lithoglyph/api/src/grpc.zig +++ /dev/null @@ -1,686 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - gRPC Handler -// -// gRPC over HTTP/2 with Protocol Buffers -// Implements the Lithoglyph gRPC service defined in proto/lithoglyph.proto - -const std = @import("std"); -const config = @import("config.zig"); -const bridge = @import("bridge_client.zig"); - -const log = std.log.scoped(.grpc); - -pub fn handleRequest( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - cfg: *const config.Config, -) !void { - _ = cfg; - - const path = request.head.target; - - // gRPC uses POST with specific content-type - if (request.head.method != .POST) { - try sendGrpcError(allocator, request, 12, "Unimplemented: Only POST supported"); - return; - } - - // Check content-type - const content_type = getHeader(request, "content-type") orelse ""; - if (!std.mem.startsWith(u8, content_type, "application/grpc")) { - try sendGrpcError(allocator, request, 3, "Invalid content-type for gRPC"); - return; - } - - // Route to service method - // Path format: /grpc/lithoglyph.v1.Lithoglyph/MethodName - if (std.mem.indexOf(u8, path, "/lithoglyph.v1.Lithoglyph/")) |idx| { - const method = path[idx + "/lithoglyph.v1.Lithoglyph/".len ..]; - try routeGrpcMethod(allocator, request, method); - } else { - try sendGrpcError(allocator, request, 12, "Unknown service"); - } -} - -fn routeGrpcMethod(allocator: std.mem.Allocator, request: *std.http.Server.Request, method: []const u8) !void { - log.info("gRPC method: {s}", .{method}); - - // Read request body (gRPC frame) - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 10 * 1024 * 1024); - defer allocator.free(body); - - // Parse gRPC frame: 1 byte compression + 4 bytes length + message - if (body.len < 5) { - try sendGrpcError(allocator, request, 3, "Invalid gRPC frame"); - return; - } - - const compressed = body[0] != 0; - if (compressed) { - try sendGrpcError(allocator, request, 12, "Compression not supported"); - return; - } - - const msg_len = std.mem.readInt(u32, body[1..5], .big); - if (body.len < 5 + msg_len) { - try sendGrpcError(allocator, request, 3, "Incomplete gRPC message"); - return; - } - - const msg_data = body[5 .. 5 + msg_len]; - - // Route to method handler - if (std.mem.eql(u8, method, "Query")) { - try handleQuery(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "ListCollections")) { - try handleListCollections(allocator, request); - } else if (std.mem.eql(u8, method, "GetCollection")) { - try handleGetCollection(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "CreateCollection")) { - try handleCreateCollection(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "GetJournal")) { - try handleGetJournal(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "DiscoverDependencies")) { - try handleDiscoverDependencies(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "AnalyzeNormalForm")) { - try handleAnalyzeNormalForm(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "StartMigration")) { - try handleStartMigration(allocator, request, msg_data); - } else if (std.mem.eql(u8, method, "Health")) { - try handleHealth(allocator, request); - } else { - try sendGrpcError(allocator, request, 12, "Unimplemented method"); - } -} - -// ============================================================================= -// Protobuf Encoding Helpers -// ============================================================================= - -const ProtobufEncoder = struct { - buffer: std.ArrayList(u8) = .empty, - allocator: std.mem.Allocator, - - fn init(allocator: std.mem.Allocator) ProtobufEncoder { - return .{ .allocator = allocator }; - } - - fn deinit(self: *ProtobufEncoder) void { - self.buffer.deinit(self.allocator); - } - - fn writeVarint(self: *ProtobufEncoder, value: u64) !void { - var v = value; - while (v >= 0x80) { - try self.buffer.append(self.allocator, @as(u8, @truncate(v)) | 0x80); - v >>= 7; - } - try self.buffer.append(self.allocator, @as(u8, @truncate(v))); - } - - fn writeTag(self: *ProtobufEncoder, field: u32, wire_type: u3) !void { - try self.writeVarint(@as(u64, field) << 3 | wire_type); - } - - fn writeString(self: *ProtobufEncoder, field: u32, value: []const u8) !void { - try self.writeTag(field, 2); // Length-delimited - try self.writeVarint(value.len); - try self.buffer.appendSlice(self.allocator, value); - } - - fn writeInt64(self: *ProtobufEncoder, field: u32, value: i64) !void { - try self.writeTag(field, 0); // Varint - try self.writeVarint(@bitCast(value)); - } - - fn writeUint64(self: *ProtobufEncoder, field: u32, value: u64) !void { - try self.writeTag(field, 0); // Varint - try self.writeVarint(value); - } - - fn writeUint32(self: *ProtobufEncoder, field: u32, value: u32) !void { - try self.writeTag(field, 0); // Varint - try self.writeVarint(value); - } - - fn writeBool(self: *ProtobufEncoder, field: u32, value: bool) !void { - try self.writeTag(field, 0); // Varint - try self.writeVarint(if (value) 1 else 0); - } - - fn writeEnum(self: *ProtobufEncoder, field: u32, value: i32) !void { - try self.writeTag(field, 0); // Varint - try self.writeVarint(@bitCast(@as(i64, value))); - } - - fn writeBytes(self: *ProtobufEncoder, field: u32, value: []const u8) !void { - try self.writeTag(field, 2); // Length-delimited - try self.writeVarint(value.len); - try self.buffer.appendSlice(self.allocator, value); - } - - fn writeMessage(self: *ProtobufEncoder, field: u32, msg: []const u8) !void { - try self.writeTag(field, 2); // Length-delimited - try self.writeVarint(msg.len); - try self.buffer.appendSlice(self.allocator, msg); - } - - fn finish(self: *ProtobufEncoder) []const u8 { - return self.buffer.items; - } -}; - -const ProtobufDecoder = struct { - data: []const u8, - pos: usize, - - fn init(data: []const u8) ProtobufDecoder { - return .{ .data = data, .pos = 0 }; - } - - fn readVarint(self: *ProtobufDecoder) !u64 { - var result: u64 = 0; - var shift: u6 = 0; - while (self.pos < self.data.len) { - const b = self.data[self.pos]; - self.pos += 1; - result |= @as(u64, b & 0x7F) << shift; - if (b < 0x80) return result; - shift += 7; - if (shift >= 64) return error.VarintOverflow; - } - return error.UnexpectedEof; - } - - fn readTag(self: *ProtobufDecoder) !struct { field: u32, wire_type: u3 } { - const v = try self.readVarint(); - return .{ - .field = @truncate(v >> 3), - .wire_type = @truncate(v & 0x7), - }; - } - - fn readString(self: *ProtobufDecoder) ![]const u8 { - const len = try self.readVarint(); - if (self.pos + len > self.data.len) return error.UnexpectedEof; - const result = self.data[self.pos .. self.pos + @as(usize, @intCast(len))]; - self.pos += @intCast(len); - return result; - } - - fn skipField(self: *ProtobufDecoder, wire_type: u3) !void { - switch (wire_type) { - 0 => _ = try self.readVarint(), // Varint - 1 => self.pos += 8, // 64-bit - 2 => { // Length-delimited - const len = try self.readVarint(); - self.pos += @intCast(len); - }, - 5 => self.pos += 4, // 32-bit - else => return error.UnknownWireType, - } - } - - fn hasMore(self: *ProtobufDecoder) bool { - return self.pos < self.data.len; - } -}; - -// ============================================================================= -// gRPC Method Handlers -// ============================================================================= - -fn handleQuery(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - // Parse QueryRequest: gql (1), explain (2), analyze (3), verbose (4), provenance (5) - var decoder = ProtobufDecoder.init(msg_data); - var gql: []const u8 = ""; - var explain = false; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => gql = try decoder.readString(), // gql - 2 => explain = (try decoder.readVarint()) != 0, // explain - else => try decoder.skipField(tag.wire_type), - } - } - - if (gql.len == 0) { - try sendGrpcError(allocator, request, 3, "Missing gql field"); - return; - } - - log.info("gRPC Query: {s}", .{gql}); - - // Execute via bridge - if (explain) { - // Return explain plan - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - // QueryResponse.plan (field 2) - var plan_encoder = ProtobufEncoder.init(allocator); - defer plan_encoder.deinit(); - try plan_encoder.writeString(4, "Full scan with filter"); // rationale - try encoder.writeMessage(2, plan_encoder.finish()); - - try sendGrpcResponse(allocator, request, encoder.finish()); - return; - } - - var result = bridge.executeQuery(gql, null) catch |err| { - log.err("Query failed: {}", .{err}); - try sendGrpcError(allocator, request, 13, "Query execution failed"); - return; - }; - defer result.deinit(allocator); - - // Build QueryResponse - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeString(1, result.data); // rows (JSON for now) - try encoder.writeUint64(3, result.rows_affected); // row_count - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleListCollections(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - const collections = bridge.listCollections() catch |err| { - log.err("ListCollections failed: {}", .{err}); - try sendGrpcError(allocator, request, 13, "Failed to list collections"); - return; - }; - defer allocator.free(collections); - - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - // ListCollectionsResponse: collections (1), total (2) - for (collections) |col| { - var col_encoder = ProtobufEncoder.init(allocator); - defer col_encoder.deinit(); - try col_encoder.writeString(1, col.name); // name - try col_encoder.writeEnum(2, 0); // type = DOCUMENT - try col_encoder.writeUint64(4, col.document_count); // document_count - try encoder.writeMessage(1, col_encoder.finish()); - } - try encoder.writeUint32(2, @intCast(collections.len)); // total - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleGetCollection(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - var decoder = ProtobufDecoder.init(msg_data); - var name: []const u8 = ""; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => name = try decoder.readString(), - else => try decoder.skipField(tag.wire_type), - } - } - - const collection = bridge.getCollection(name) catch |err| { - log.err("GetCollection failed: {}", .{err}); - try sendGrpcError(allocator, request, 13, "Failed to get collection"); - return; - }; - - if (collection) |col| { - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - try encoder.writeString(1, col.name); - try encoder.writeEnum(2, 0); // DOCUMENT - try encoder.writeUint64(4, col.document_count); - try sendGrpcResponse(allocator, request, encoder.finish()); - } else { - try sendGrpcError(allocator, request, 5, "Collection not found"); - } -} - -fn handleCreateCollection(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - var decoder = ProtobufDecoder.init(msg_data); - var name: []const u8 = ""; - var schema: []const u8 = "{}"; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => name = try decoder.readString(), - 3 => schema = try decoder.readString(), // schema_json - else => try decoder.skipField(tag.wire_type), - } - } - - bridge.createCollection(name, schema) catch |err| { - log.err("CreateCollection failed: {}", .{err}); - const msg: []const u8 = switch (err) { - error.NotImplemented => "Collection creation not yet implemented", - }; - try sendGrpcError(allocator, request, 12, msg); - return; - }; - - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - try encoder.writeString(1, name); - try encoder.writeEnum(2, 0); // DOCUMENT - try encoder.writeUint64(4, 0); // document_count - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleGetJournal(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - var decoder = ProtobufDecoder.init(msg_data); - var since: u64 = 0; - var limit: u32 = 100; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => since = try decoder.readVarint(), - 2 => limit = @truncate(try decoder.readVarint()), - else => try decoder.skipField(tag.wire_type), - } - } - - const entries = bridge.getJournal(since, limit) catch |err| { - log.err("GetJournal failed: {}", .{err}); - try sendGrpcError(allocator, request, 13, "Failed to get journal"); - return; - }; - defer allocator.free(entries); - - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - for (entries) |entry| { - var entry_encoder = ProtobufEncoder.init(allocator); - defer entry_encoder.deinit(); - try entry_encoder.writeUint64(1, entry.sequence); - try entry_encoder.writeString(2, entry.timestamp); - try entry_encoder.writeString(3, entry.operation); - if (entry.collection) |col| { - try entry_encoder.writeString(4, col); - } - try encoder.writeMessage(1, entry_encoder.finish()); - } - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleDiscoverDependencies(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - // Parse DiscoverDependenciesRequest: collection (1), sample_size (2) - var decoder = ProtobufDecoder.init(msg_data); - var collection: []const u8 = ""; - var sample_size: u32 = 1000; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => collection = try decoder.readString(), - 2 => sample_size = @truncate(try decoder.readVarint()), - else => try decoder.skipField(tag.wire_type), - } - } - - if (collection.len == 0) { - try sendGrpcError(allocator, request, 3, "Missing collection field"); - return; - } - - const deps = bridge.discoverDependencies(collection, sample_size) catch |err| { - log.err("DiscoverDependencies failed for {s}: {}", .{ collection, err }); - const msg: []const u8 = switch (err) { - error.NotImplemented => "Dependency discovery not yet implemented in bridge", - }; - const code: u8 = switch (err) { - error.NotImplemented => 12, - }; - try sendGrpcError(allocator, request, code, msg); - return; - }; - defer allocator.free(deps); - - // Build DiscoverDependenciesResponse - // collection (1), dependencies repeated (2) - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeString(1, collection); - for (deps) |dep| { - // Encode each FunctionalDependency as a sub-message - // FunctionalDependency: determinant repeated (1), dependent (2), confidence (3) - var dep_encoder = ProtobufEncoder.init(allocator); - defer dep_encoder.deinit(); - for (dep.determinant) |det| { - try dep_encoder.writeString(1, det); - } - try dep_encoder.writeString(2, dep.dependent); - // Encode confidence as fixed32 (IEEE 754 float) - try dep_encoder.writeTag(3, 5); // wire type 5 = 32-bit - const conf_bits: u32 = @bitCast(dep.confidence); - try dep_encoder.buffer.appendSlice(dep_encoder.allocator, &std.mem.toBytes(conf_bits)); - try encoder.writeMessage(2, dep_encoder.finish()); - } - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleAnalyzeNormalForm(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - // Parse AnalyzeNormalFormRequest: collection (1) - var decoder = ProtobufDecoder.init(msg_data); - var collection: []const u8 = ""; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => collection = try decoder.readString(), - else => try decoder.skipField(tag.wire_type), - } - } - - if (collection.len == 0) { - try sendGrpcError(allocator, request, 3, "Missing collection field"); - return; - } - - const analysis = bridge.analyzeNormalForm(collection) catch |err| { - log.err("AnalyzeNormalForm failed for {s}: {}", .{ collection, err }); - const msg: []const u8 = switch (err) { - error.NotImplemented => "Normal form analysis not yet implemented in bridge", - }; - const code: u8 = switch (err) { - error.NotImplemented => 12, - }; - try sendGrpcError(allocator, request, code, msg); - return; - }; - - // Build AnalyzeNormalFormResponse - // collection (1), current_form enum (2), violations repeated string (3), suggestions repeated string (4) - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeString(1, collection); - - // Map current_form string to enum value - // NormalForm enum: UNF=0, 1NF=1, 2NF=2, 3NF=3, BCNF=4, 4NF=5, 5NF=6 - const form_enum: i32 = if (std.mem.eql(u8, analysis.current_form, "UNF")) - 0 - else if (std.mem.eql(u8, analysis.current_form, "1NF")) - 1 - else if (std.mem.eql(u8, analysis.current_form, "2NF")) - 2 - else if (std.mem.eql(u8, analysis.current_form, "3NF")) - 3 - else if (std.mem.eql(u8, analysis.current_form, "BCNF")) - 4 - else if (std.mem.eql(u8, analysis.current_form, "4NF")) - 5 - else if (std.mem.eql(u8, analysis.current_form, "5NF")) - 6 - else - 0; // Default to UNF for unknown forms - - try encoder.writeEnum(2, form_enum); - - for (analysis.violations) |violation| { - try encoder.writeString(3, violation); - } - for (analysis.suggestions) |suggestion| { - try encoder.writeString(4, suggestion); - } - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleStartMigration(allocator: std.mem.Allocator, request: *std.http.Server.Request, msg_data: []const u8) !void { - // Parse StartMigrationRequest: collection (1), target_schema (2) - var decoder = ProtobufDecoder.init(msg_data); - var collection: []const u8 = ""; - var target_schema: []const u8 = "{}"; - - while (decoder.hasMore()) { - const tag = try decoder.readTag(); - switch (tag.field) { - 1 => collection = try decoder.readString(), - 2 => target_schema = try decoder.readString(), - else => try decoder.skipField(tag.wire_type), - } - } - - if (collection.len == 0) { - try sendGrpcError(allocator, request, 3, "Missing collection field"); - return; - } - - const migration = bridge.startMigration(collection, target_schema) catch |err| { - log.err("StartMigration failed for {s}: {}", .{ collection, err }); - const msg: []const u8 = switch (err) { - error.NotImplemented => "Migration not yet implemented in bridge", - }; - const code: u8 = switch (err) { - error.NotImplemented => 12, - }; - try sendGrpcError(allocator, request, code, msg); - return; - }; - - // Build StartMigrationResponse - // id (1), phase enum (2), collection (3), narrative (4) - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeString(1, migration.id); - - // MigrationPhase enum: ANNOUNCE=0, SHADOW=1, SHADOW_COMPLETE=2, COMPLETE=3, ABORTED=4 - const phase_enum: i32 = switch (migration.state) { - .announced => 0, - .shadow_running => 1, - .shadow_complete => 2, - .committed => 3, - .aborted => 4, - }; - try encoder.writeEnum(2, phase_enum); - try encoder.writeString(3, migration.source_collection); - try encoder.writeString(4, migration.created_at); - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -fn handleHealth(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - const health = bridge.getHealth(); - - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - // HealthResponse: status (1), version (2), uptime_seconds (3) - try encoder.writeEnum(1, if (std.mem.eql(u8, health.status, "healthy")) 0 else 1); - try encoder.writeString(2, health.version); - try encoder.writeUint64(3, health.uptime_seconds); - - try sendGrpcResponse(allocator, request, encoder.finish()); -} - -// ============================================================================= -// gRPC Response Helpers -// ============================================================================= - -fn sendGrpcResponse(allocator: std.mem.Allocator, request: *std.http.Server.Request, data: []const u8) !void { - // gRPC uses length-prefixed messages - // Format: 1 byte compression flag + 4 bytes length + data - var frame = try allocator.alloc(u8, 5 + data.len); - defer allocator.free(frame); - - frame[0] = 0; // No compression - std.mem.writeInt(u32, frame[1..5], @intCast(data.len), .big); - @memcpy(frame[5..], data); - - request.respond(frame, .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/grpc+proto" }, - .{ .name = "grpc-status", .value = "0" }, - }, - }) catch {}; -} - -fn sendGrpcError(allocator: std.mem.Allocator, request: *std.http.Server.Request, code: u8, message: []const u8) !void { - _ = allocator; - - var code_buf: [8]u8 = undefined; - const code_str = std.fmt.bufPrint(&code_buf, "{d}", .{code}) catch "0"; - - // gRPC conveys error details via grpc-status and grpc-message trailers - request.respond("", .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/grpc+proto" }, - .{ .name = "grpc-status", .value = code_str }, - .{ .name = "grpc-message", .value = message }, - }, - }) catch {}; -} - -fn getHeader(request: *std.http.Server.Request, name: []const u8) ?[]const u8 { - var iter = request.iterateHeaders(); - while (iter.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, name)) { - return header.value; - } - } - return null; -} - -test "grpc path parsing" { - const path = "/grpc/lithoglyph.v1.Lithoglyph/Query"; - if (std.mem.indexOf(u8, path, "/lithoglyph.v1.Lithoglyph/")) |idx| { - const method = path[idx + "/lithoglyph.v1.Lithoglyph/".len ..]; - try std.testing.expectEqualStrings("Query", method); - } else { - return error.TestFailed; - } -} - -test "protobuf varint encoding" { - const allocator = std.testing.allocator; - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeVarint(150); - try std.testing.expectEqualSlices(u8, &[_]u8{ 0x96, 0x01 }, encoder.finish()); -} - -test "protobuf string encoding" { - const allocator = std.testing.allocator; - var encoder = ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - try encoder.writeString(1, "test"); - // Field 1, wire type 2 = 0x0a, length 4 - try std.testing.expectEqualSlices(u8, &[_]u8{ 0x0a, 0x04, 't', 'e', 's', 't' }, encoder.finish()); -} diff --git a/lithoglyph/api/src/integration_tests.zig b/lithoglyph/api/src/integration_tests.zig deleted file mode 100644 index 6321cb54..00000000 --- a/lithoglyph/api/src/integration_tests.zig +++ /dev/null @@ -1,325 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Integration Tests -// -// End-to-end tests for REST, gRPC, and GraphQL endpoints -// Tests API server with mocked Form.Bridge responses - -const std = @import("std"); -const rest = @import("rest.zig"); -const grpc = @import("grpc.zig"); -const graphql = @import("graphql.zig"); -const websocket = @import("websocket.zig"); -const bridge = @import("bridge_client.zig"); -const config = @import("config.zig"); - -// ============================================================================= -// Test Utilities -// ============================================================================= - -fn createTestAllocator() std.mem.Allocator { - return std.testing.allocator; -} - -// ============================================================================= -// REST API Integration Tests -// ============================================================================= - -test "REST health endpoint returns valid JSON" { - const allocator = createTestAllocator(); - - // Verify bridge health response format - const health = bridge.getHealth(); - try std.testing.expect(health.status.len > 0); - try std.testing.expect(health.version.len > 0); - try std.testing.expect(health.uptime_seconds >= 0); - - _ = allocator; -} - -test "REST collections endpoint returns valid structure" { - const allocator = createTestAllocator(); - - // Verify bridge collections response format - const collections = try bridge.getCollections(allocator); - defer allocator.free(collections); - - // Should be valid JSON array - const parsed = try std.json.parseFromSlice(std.json.Value, allocator, collections, .{}); - defer parsed.deinit(); - - try std.testing.expect(parsed.value == .array); -} - -test "REST query execution returns valid response" { - const allocator = createTestAllocator(); - - // Test GQL query execution - const result = try bridge.executeQuery(allocator, "SELECT * FROM test"); - defer allocator.free(result); - - // Should be valid JSON with rows - const parsed = try std.json.parseFromSlice(std.json.Value, allocator, result, .{}); - defer parsed.deinit(); - - try std.testing.expect(parsed.value == .object); -} - -// ============================================================================= -// gRPC Integration Tests -// ============================================================================= - -test "gRPC protobuf encoder creates valid frames" { - const allocator = createTestAllocator(); - var encoder = grpc.ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - // Write a string field (field 1, wire type 2 = length-delimited) - try encoder.writeString(1, "test_value"); - - const encoded = encoder.getEncoded(); - try std.testing.expect(encoded.len > 0); - - // First byte should be tag (field 1 << 3 | wire type 2 = 10 = 0x0A) - try std.testing.expectEqual(@as(u8, 0x0A), encoded[0]); -} - -test "gRPC protobuf encoder handles varints correctly" { - const allocator = createTestAllocator(); - var encoder = grpc.ProtobufEncoder.init(allocator); - defer encoder.deinit(); - - // Test small varint (single byte) - try encoder.writeVarint(127); - try std.testing.expectEqual(@as(usize, 1), encoder.getEncoded().len); - - // Reset and test multi-byte varint - encoder.reset(); - try encoder.writeVarint(300); - // 300 = 0b100101100 = 0xAC 0x02 in varint encoding - try std.testing.expectEqual(@as(usize, 2), encoder.getEncoded().len); -} - -test "gRPC protobuf decoder parses valid messages" { - const allocator = createTestAllocator(); - - // Create a simple message: field 1 = "hello" - var encoder = grpc.ProtobufEncoder.init(allocator); - defer encoder.deinit(); - try encoder.writeString(1, "hello"); - - const encoded = encoder.getEncoded(); - - // Now decode it - var decoder = grpc.ProtobufDecoder.init(encoded); - - const field = try decoder.readField(); - try std.testing.expectEqual(@as(u32, 1), field.field_number); - try std.testing.expectEqual(@as(u3, 2), field.wire_type); - - const value = try decoder.readString(allocator); - defer allocator.free(value); - - try std.testing.expectEqualStrings("hello", value); -} - -test "gRPC frame encoding creates valid structure" { - const allocator = createTestAllocator(); - - const message = "test message"; - const frame = try grpc.encodeGrpcFrame(allocator, message); - defer allocator.free(frame); - - // gRPC frame: 1 byte compression + 4 bytes length + message - try std.testing.expectEqual(@as(usize, 5 + message.len), frame.len); - - // First byte should be 0 (no compression) - try std.testing.expectEqual(@as(u8, 0), frame[0]); - - // Next 4 bytes should be message length in big endian - const len = std.mem.readInt(u32, frame[1..5], .big); - try std.testing.expectEqual(@as(u32, @intCast(message.len)), len); - - // Rest should be the message - try std.testing.expectEqualStrings(message, frame[5..]); -} - -// ============================================================================= -// GraphQL Integration Tests -// ============================================================================= - -test "GraphQL request parsing handles valid JSON" { - const allocator = createTestAllocator(); - - const body = - \\{"query": "{ health { status } }", "variables": null} - ; - - const parsed = try std.json.parseFromSlice(graphql.GraphQLRequest, allocator, body, .{}); - defer parsed.deinit(); - - try std.testing.expect(std.mem.indexOf(u8, parsed.value.query, "health") != null); -} - -test "GraphQL introspection detection works" { - const query1 = "{ __schema { types { name } } }"; - const query2 = "{ __type(name: \"Query\") { fields { name } } }"; - const query3 = "{ health { status } }"; - - // __schema should trigger introspection - try std.testing.expect(std.mem.indexOf(u8, query1, "__schema") != null); - try std.testing.expect(std.mem.indexOf(u8, query2, "__type") != null); - - // Regular query should not - try std.testing.expect(std.mem.indexOf(u8, query3, "__schema") == null); - try std.testing.expect(std.mem.indexOf(u8, query3, "__type") == null); -} - -// ============================================================================= -// WebSocket Integration Tests -// ============================================================================= - -test "WebSocket accept key computation follows RFC 6455" { - const allocator = createTestAllocator(); - - // Example from RFC 6455 Section 1.3 - const key = "dGhlIHNhbXBsZSBub25jZQ=="; - const expected = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="; - - const accept = try websocket.computeAcceptKey(allocator, key); - defer allocator.free(accept); - - try std.testing.expectEqualStrings(expected, accept); -} - -test "WebSocket frame encoding creates valid frames" { - const allocator = createTestAllocator(); - - // Small payload (< 126 bytes) - const frame = try websocket.encodeFrame(allocator, .text, "Hello"); - defer allocator.free(frame); - - // First byte: FIN (0x80) | opcode (0x01 for text) = 0x81 - try std.testing.expectEqual(@as(u8, 0x81), frame[0]); - - // Second byte: payload length (5) - try std.testing.expectEqual(@as(u8, 5), frame[1]); - - // Payload - try std.testing.expectEqualStrings("Hello", frame[2..7]); -} - -test "WebSocket frame encoding handles medium payloads" { - const allocator = createTestAllocator(); - - // Medium payload (126-65535 bytes) - var payload: [200]u8 = undefined; - @memset(&payload, 'A'); - - const frame = try websocket.encodeFrame(allocator, .text, &payload); - defer allocator.free(frame); - - // First byte: FIN | text - try std.testing.expectEqual(@as(u8, 0x81), frame[0]); - - // Second byte: 126 indicates 16-bit length follows - try std.testing.expectEqual(@as(u8, 126), frame[1]); - - // Next 2 bytes: length in big endian - const len = std.mem.readInt(u16, frame[2..4], .big); - try std.testing.expectEqual(@as(u16, 200), len); -} - -test "WebSocket subscription message creation" { - const allocator = createTestAllocator(); - - // Connection ack message - const msg = try websocket.createSubscriptionMessage(allocator, "connection_ack", null, null); - defer allocator.free(msg); - - try std.testing.expectEqualStrings("{\"type\":\"connection_ack\"}", msg); -} - -test "WebSocket subscription message with id and payload" { - const allocator = createTestAllocator(); - - const msg = try websocket.createSubscriptionMessage( - allocator, - "next", - "sub-123", - "{\"data\":{\"journalStream\":{\"seq\":42}}}", - ); - defer allocator.free(msg); - - // Should contain type, id, and payload - try std.testing.expect(std.mem.indexOf(u8, msg, "\"type\":\"next\"") != null); - try std.testing.expect(std.mem.indexOf(u8, msg, "\"id\":\"sub-123\"") != null); - try std.testing.expect(std.mem.indexOf(u8, msg, "\"payload\":") != null); -} - -test "WebSocket connection management" { - const allocator = createTestAllocator(); - - var conn = websocket.Connection.init(allocator); - defer conn.deinit(); - - try std.testing.expect(conn.is_open); - try std.testing.expectEqual(@as(usize, 0), conn.subscriptions.count()); - - // Add subscription - try conn.addSubscription("sub-1", .journal_stream); - try std.testing.expectEqual(@as(usize, 1), conn.subscriptions.count()); - - // Remove subscription - conn.removeSubscription("sub-1"); - try std.testing.expectEqual(@as(usize, 0), conn.subscriptions.count()); -} - -// ============================================================================= -// Bridge Client Integration Tests -// ============================================================================= - -test "Bridge client health check returns valid response" { - const health = bridge.getHealth(); - - // Should have valid fields - try std.testing.expect(health.status.len > 0); - try std.testing.expect(health.version.len > 0); -} - -test "Bridge client CBOR encoding produces valid output" { - const allocator = createTestAllocator(); - - // Test CBOR map encoding - var cbor: std.ArrayList(u8) = .empty; - defer cbor.deinit(allocator); - - // Write a simple CBOR map: {1: "test"} - try cbor.append(0xA1); // Map with 1 item - try cbor.append(0x01); // Key: 1 - try cbor.append(0x64); // Text string of length 4 - try cbor.appendSlice("test"); - - try std.testing.expectEqual(@as(usize, 7), cbor.items.len); -} - -// ============================================================================= -// Configuration Tests -// ============================================================================= - -test "Config default values are sensible" { - const cfg = config.Config{}; - - try std.testing.expectEqual(@as(u16, 8080), cfg.port); - try std.testing.expectEqualStrings("0.0.0.0", cfg.host); - try std.testing.expect(cfg.max_connections > 0); -} - -test "Config environment variable parsing" { - // This test verifies the config module can parse environment - // Note: Actual env vars won't be set in test environment - const cfg = config.Config{}; - - // Default values should be used when env vars not set - try std.testing.expect(cfg.port > 0); - try std.testing.expect(cfg.host.len > 0); -} diff --git a/lithoglyph/api/src/main.zig b/lithoglyph/api/src/main.zig deleted file mode 100644 index 36fbac23..00000000 --- a/lithoglyph/api/src/main.zig +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Multi-Protocol (REST, gRPC, GraphQL) - -const std = @import("std"); -const builtin = @import("builtin"); - -const config = @import("config.zig"); -const router = @import("router.zig"); -const rest = @import("rest.zig"); -const grpc = @import("grpc.zig"); -const graphql = @import("graphql.zig"); -const metrics = @import("metrics.zig"); -const auth = @import("auth.zig"); -const bridge_client = @import("bridge_client.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const log = std.log.scoped(.lithoglyph_server); - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Parse configuration - const cfg = try config.load(allocator); - defer cfg.deinit(); - - log.info("Lithoglyph API Server v{s}", .{cfg.version}); - log.info("Listening on {s}:{d}", .{ cfg.host, cfg.port }); - - // Initialize metrics - try metrics.init(allocator); - defer metrics.deinit(); - - // Initialize authentication - try auth.init(allocator, cfg); - defer auth.deinit(); - - // Initialize Form.Bridge connection - bridge_client.init(allocator, cfg) catch |err| { - log.warn("Failed to initialize bridge client: {} - running in degraded mode", .{err}); - }; - defer bridge_client.deinit(); - - // Create TCP listener (Zig 0.15.2 API) - const address = try std.net.Address.parseIp(cfg.host, cfg.port); - var tcp_server = try address.listen(.{ - .reuse_address = true, - }); - defer tcp_server.deinit(); - - log.info("Server started successfully", .{}); - log.info(" REST API: http://{s}:{d}/v1/", .{ cfg.host, cfg.port }); - log.info(" gRPC: http://{s}:{d}/grpc/", .{ cfg.host, cfg.port }); - log.info(" GraphQL: http://{s}:{d}/graphql", .{ cfg.host, cfg.port }); - log.info(" Health: http://{s}:{d}/v1/health", .{ cfg.host, cfg.port }); - log.info(" Metrics: http://{s}:{d}/v1/metrics", .{ cfg.host, cfg.port }); - - // Accept connections - while (true) { - const conn = tcp_server.accept() catch |err| { - log.err("Accept error: {}", .{err}); - continue; - }; - - // Spawn handler thread - _ = try std.Thread.spawn(.{}, handleConnection, .{ allocator, conn, cfg }); - } -} - -fn handleConnection(allocator: std.mem.Allocator, conn: std.net.Server.Connection, cfg: *const config.Config) void { - defer conn.stream.close(); - - // Create HTTP server from connection stream (Zig 0.15.2 API) - var recv_buffer: [8192]u8 = undefined; - var send_buffer: [8192]u8 = undefined; - var conn_reader = conn.stream.reader(&recv_buffer); - var conn_writer = conn.stream.writer(&send_buffer); - var http = std.http.Server.init(conn_reader.interface(), &conn_writer.interface); - - while (http.reader.state == .ready) { - var request = http.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => { - log.err("Receive error: {s}", .{@errorName(err)}); - return; - }, - }; - - handleRequest(allocator, &request, cfg) catch |err| { - log.err("Handle error: {s}", .{@errorName(err)}); - return; - }; - } -} - -fn handleRequest( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - cfg: *const config.Config, -) !void { - const start_time = std.time.nanoTimestamp(); - defer { - const elapsed = std.time.nanoTimestamp() - start_time; - metrics.recordLatency(@intCast(elapsed)); - } - - metrics.incrementRequests(); - - const path = request.head.target; - - // Route to appropriate handler - if (std.mem.startsWith(u8, path, "/v1/")) { - // REST API - try rest.handleRequest(allocator, request, cfg); - } else if (std.mem.startsWith(u8, path, "/grpc/")) { - // gRPC (HTTP/2 with protobuf) - try grpc.handleRequest(allocator, request, cfg); - } else if (std.mem.startsWith(u8, path, "/graphql")) { - // GraphQL - try graphql.handleRequest(allocator, request, cfg); - } else { - // 404 - try sendNotFound(request); - } -} - -fn sendNotFound(request: *std.http.Server.Request) !void { - const body = - \\{"error":"not_found","message":"Resource not found"} - ; - - request.respond(body, .{ - .status = .not_found, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -test "server initialization" { - // Basic test to ensure compilation - const allocator = std.testing.allocator; - _ = allocator; -} diff --git a/lithoglyph/api/src/metrics.zig b/lithoglyph/api/src/metrics.zig deleted file mode 100644 index 09ebffa3..00000000 --- a/lithoglyph/api/src/metrics.zig +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - Prometheus Metrics - -const std = @import("std"); - -var allocator: std.mem.Allocator = undefined; -var start_time: i64 = 0; - -// Counters -var requests_total: u64 = 0; -var requests_by_status: [5]u64 = .{ 0, 0, 0, 0, 0 }; // 2xx, 3xx, 4xx, 5xx, other -var requests_by_protocol: [3]u64 = .{ 0, 0, 0 }; // REST, gRPC, GraphQL - -// Histograms (simplified - just buckets) -var latency_buckets: [10]u64 = .{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -const latency_bounds = [_]i64{ 1_000_000, 5_000_000, 10_000_000, 25_000_000, 50_000_000, 100_000_000, 250_000_000, 500_000_000, 1_000_000_000, std.math.maxInt(i64) }; // ns - -// Gauges -var active_connections: u64 = 0; -var active_migrations: u64 = 0; - -pub fn init(alloc: std.mem.Allocator) !void { - allocator = alloc; - start_time = std.time.timestamp(); -} - -pub fn deinit() void { - // Nothing to clean up -} - -pub fn incrementRequests() void { - _ = @atomicRmw(u64, &requests_total, .Add, 1, .seq_cst); -} - -pub fn recordStatus(status: std.http.Status) void { - const code = @intFromEnum(status); - const bucket: usize = if (code >= 200 and code < 300) - 0 - else if (code >= 300 and code < 400) - 1 - else if (code >= 400 and code < 500) - 2 - else if (code >= 500 and code < 600) - 3 - else - 4; - - _ = @atomicRmw(u64, &requests_by_status[bucket], .Add, 1, .seq_cst); -} - -pub fn recordProtocol(protocol: Protocol) void { - _ = @atomicRmw(u64, &requests_by_protocol[@intFromEnum(protocol)], .Add, 1, .seq_cst); -} - -pub const Protocol = enum(u2) { - rest = 0, - grpc = 1, - graphql = 2, -}; - -pub fn recordLatency(latency_ns: i64) void { - for (latency_bounds, 0..) |bound, i| { - if (latency_ns <= bound) { - _ = @atomicRmw(u64, &latency_buckets[i], .Add, 1, .seq_cst); - break; - } - } -} - -pub fn incrementConnections() void { - _ = @atomicRmw(u64, &active_connections, .Add, 1, .seq_cst); -} - -pub fn decrementConnections() void { - _ = @atomicRmw(u64, &active_connections, .Sub, 1, .seq_cst); -} - -pub fn setActiveMigrations(count: u64) void { - @atomicStore(u64, &active_migrations, count, .seq_cst); -} - -pub fn getPrometheus(alloc: std.mem.Allocator) ![]const u8 { - var buffer: std.ArrayList(u8) = .empty; - const writer = buffer.writer(alloc); - - const uptime = std.time.timestamp() - start_time; - - // Write metrics in Prometheus format - try writer.print( - \\# HELP lithoglyph_requests_total Total number of requests - \\# TYPE lithoglyph_requests_total counter - \\lithoglyph_requests_total {d} - \\ - \\# HELP lithoglyph_requests_by_status Requests by HTTP status code range - \\# TYPE lithoglyph_requests_by_status counter - \\lithoglyph_requests_by_status{{status="2xx"}} {d} - \\lithoglyph_requests_by_status{{status="3xx"}} {d} - \\lithoglyph_requests_by_status{{status="4xx"}} {d} - \\lithoglyph_requests_by_status{{status="5xx"}} {d} - \\ - \\# HELP lithoglyph_requests_by_protocol Requests by protocol - \\# TYPE lithoglyph_requests_by_protocol counter - \\lithoglyph_requests_by_protocol{{protocol="rest"}} {d} - \\lithoglyph_requests_by_protocol{{protocol="grpc"}} {d} - \\lithoglyph_requests_by_protocol{{protocol="graphql"}} {d} - \\ - \\# HELP lithoglyph_request_duration_seconds Request latency histogram - \\# TYPE lithoglyph_request_duration_seconds histogram - \\lithoglyph_request_duration_seconds_bucket{{le="0.001"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.005"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.01"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.025"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.05"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.1"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.25"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="0.5"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="1.0"}} {d} - \\lithoglyph_request_duration_seconds_bucket{{le="+Inf"}} {d} - \\ - \\# HELP lithoglyph_active_connections Current number of active connections - \\# TYPE lithoglyph_active_connections gauge - \\lithoglyph_active_connections {d} - \\ - \\# HELP lithoglyph_active_migrations Current number of active migrations - \\# TYPE lithoglyph_active_migrations gauge - \\lithoglyph_active_migrations {d} - \\ - \\# HELP lithoglyph_uptime_seconds Server uptime in seconds - \\# TYPE lithoglyph_uptime_seconds gauge - \\lithoglyph_uptime_seconds {d} - \\ - \\# HELP lithoglyph_info Server version information - \\# TYPE lithoglyph_info gauge - \\lithoglyph_info{{version="0.0.4"}} 1 - \\ - , .{ - requests_total, - requests_by_status[0], - requests_by_status[1], - requests_by_status[2], - requests_by_status[3], - requests_by_protocol[0], - requests_by_protocol[1], - requests_by_protocol[2], - latency_buckets[0], - latency_buckets[0] + latency_buckets[1], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4] + latency_buckets[5], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4] + latency_buckets[5] + latency_buckets[6], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4] + latency_buckets[5] + latency_buckets[6] + latency_buckets[7], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4] + latency_buckets[5] + latency_buckets[6] + latency_buckets[7] + latency_buckets[8], - latency_buckets[0] + latency_buckets[1] + latency_buckets[2] + latency_buckets[3] + latency_buckets[4] + latency_buckets[5] + latency_buckets[6] + latency_buckets[7] + latency_buckets[8] + latency_buckets[9], - active_connections, - active_migrations, - uptime, - }); - - return buffer.toOwnedSlice(alloc); -} - -test "metrics increment" { - try init(std.testing.allocator); - defer deinit(); - - incrementRequests(); - incrementRequests(); - - try std.testing.expectEqual(@as(u64, 2), requests_total); -} diff --git a/lithoglyph/api/src/rest.zig b/lithoglyph/api/src/rest.zig deleted file mode 100644 index cd4c4e00..00000000 --- a/lithoglyph/api/src/rest.zig +++ /dev/null @@ -1,800 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - REST Handler - -const std = @import("std"); -const json = std.json; - -const config = @import("config.zig"); -const auth = @import("auth.zig"); -const metrics = @import("metrics.zig"); -const bridge = @import("bridge_client.zig"); - -const log = std.log.scoped(.rest); - -pub fn handleRequest( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - cfg: *const config.Config, -) !void { - // Authentication check - if (cfg.require_auth) { - if (!try auth.validateRequest(request)) { - try sendUnauthorized(request); - return; - } - } - - const path = request.head.target; - const method = request.head.method; - - // Strip /v1/ prefix - const endpoint = if (std.mem.startsWith(u8, path, "/v1/")) - path[4..] - else - path; - - // Route to handler - if (std.mem.eql(u8, endpoint, "/query") or std.mem.eql(u8, endpoint, "/query/")) { - try handleQuery(allocator, request, method); - } else if (std.mem.startsWith(u8, endpoint, "/collections")) { - try handleCollections(allocator, request, method, endpoint); - } else if (std.mem.startsWith(u8, endpoint, "/journal")) { - try handleJournal(allocator, request, method); - } else if (std.mem.startsWith(u8, endpoint, "/normalize")) { - try handleNormalize(allocator, request, method, endpoint); - } else if (std.mem.startsWith(u8, endpoint, "/migrate")) { - try handleMigrate(allocator, request, method, endpoint); - } else if (std.mem.eql(u8, endpoint, "/health") or std.mem.eql(u8, endpoint, "/health/")) { - try handleHealth(allocator, request); - } else if (std.mem.eql(u8, endpoint, "/metrics") or std.mem.eql(u8, endpoint, "/metrics/")) { - try handleMetrics(allocator, request); - } else { - try sendNotFound(request); - } -} - -// ============================================================================= -// Query Handler -// ============================================================================= - -fn handleQuery( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - method: std.http.Method, -) !void { - if (method != .POST) { - try sendMethodNotAllowed(request); - return; - } - - // Read request body - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 10 * 1024 * 1024); - defer allocator.free(body); - - // Parse JSON request - const parsed = json.parseFromSlice(QueryRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - - log.info("Executing GQL: {s}", .{req.ldql}); - - // EXPLAIN mode - return query plan without execution - if (req.explain) { - const response = - \\{ - \\ "plan": { - \\ "steps": [ - \\ {"type": "scan", "collection": "articles"}, - \\ {"type": "filter", "expression": "status = 'published'"}, - \\ {"type": "limit", "count": 10} - \\ ], - \\ "estimatedCost": 150.0, - \\ "rationale": "Full scan with filter (no index on status)" - \\ }, - \\ "timing": { - \\ "parseMs": 0.5, - \\ "planMs": 1.2, - \\ "executeMs": 0.0, - \\ "totalMs": 1.7 - \\ } - \\} - ; - try sendJson(request, .ok, response); - return; - } - - // Execute via Form.Bridge - const prov = if (req.provenance) |p| bridge.QueryProvenance{ - .actor = p.actor, - .rationale = p.rationale, - } else null; - - var result = bridge.executeQuery(req.ldql, prov) catch |err| { - log.err("Query execution failed: {}", .{err}); - - // Return error response - const error_response = switch (err) { - error.NotInitialized => - \\{"error":"service_unavailable","message":"Database not initialized"} - , - error.TransactionFailed => - \\{"error":"transaction_error","message":"Failed to begin transaction"} - , - error.ApplyFailed => - \\{"error":"execution_error","message":"Query execution failed"} - , - error.CommitFailed => - \\{"error":"commit_error","message":"Failed to commit transaction"} - , - else => - \\{"error":"internal_error","message":"Internal server error"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - defer result.deinit(allocator); - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print( - \\{{"rows":{s},"rowCount":{d},"journalSeq":0, - , .{ result.data, result.rows_affected }); - - // Include provenance if present - if (result.provenance) |prov_json| { - try writer.print( - \\"provenance":{s}, - , .{prov_json}); - } - - try writer.writeAll( - \\"timing":{"parseMs":0.5,"planMs":1.2,"executeMs":3.8,"totalMs":5.5}}} - ); - - try sendJson(request, .ok, response_buffer.items); -} - -const QueryRequest = struct { - ldql: []const u8, - provenance: ?Provenance = null, - explain: bool = false, - analyze: bool = false, - verbose: bool = false, -}; - -const Provenance = struct { - actor: []const u8, - rationale: []const u8, -}; - -// ============================================================================= -// Collections Handler -// ============================================================================= - -fn handleCollections( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - method: std.http.Method, - endpoint: []const u8, -) !void { - // Check if it's a specific collection - const collection_name = extractCollectionName(endpoint); - - if (collection_name) |name| { - switch (method) { - .GET => try handleGetCollection(allocator, request, name), - .DELETE => try handleDropCollection(request, name), - else => try sendMethodNotAllowed(request), - } - } else { - switch (method) { - .GET => try handleListCollections(allocator, request), - .POST => try handleCreateCollection(allocator, request), - else => try sendMethodNotAllowed(request), - } - } -} - -fn extractCollectionName(endpoint: []const u8) ?[]const u8 { - // /collections/name -> name - const prefix = "/collections/"; - if (std.mem.startsWith(u8, endpoint, prefix) and endpoint.len > prefix.len) { - return endpoint[prefix.len..]; - } - return null; -} - -fn handleListCollections(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - const collections = bridge.listCollections() catch |err| { - log.err("Failed to list collections: {}", .{err}); - // Fall back to empty list - try sendJson(request, .ok, - \\{"collections":[],"total":0} - ); - return; - }; - defer allocator.free(collections); - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.writeAll("{\"collections\":["); - for (collections, 0..) |col, i| { - if (i > 0) try writer.writeByte(','); - try writer.print( - \\{{"name":"{s}","type":"document","documentCount":{d},"normalForm":"unknown"}} - , .{ col.name, col.document_count }); - } - try writer.print("],\"total\":{d}}}", .{collections.len}); - - try sendJson(request, .ok, response_buffer.items); -} - -fn handleGetCollection(allocator: std.mem.Allocator, request: *std.http.Server.Request, name: []const u8) !void { - const collection = bridge.getCollection(name) catch |err| { - log.err("Failed to get collection {s}: {}", .{ name, err }); - try sendJson(request, .internal_server_error, - \\{"error":"internal_error","message":"Failed to retrieve collection"} - ); - return; - }; - - if (collection) |col| { - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print( - \\{{"name":"{s}","type":"document","schema":{{"fields":[],"constraints":[]}},"documentCount":{d},"normalForm":"unknown"}} - , .{ col.name, col.document_count }); - - try sendJson(request, .ok, response_buffer.items); - } else { - try sendNotFound(request); - } -} - -fn handleCreateCollection(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - // Read request body - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 1024 * 1024); - defer allocator.free(body); - - // Parse JSON request - const parsed = json.parseFromSlice(CreateCollectionRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - - bridge.createCollection(req.name, req.schema orelse "{}") catch |err| { - log.err("Failed to create collection: {}", .{err}); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"not_implemented","message":"Collection creation not yet implemented"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print( - \\{{"name":"{s}","type":"document","documentCount":0,"normalForm":"unknown"}} - , .{req.name}); - - try sendJson(request, .created, response_buffer.items); -} - -const CreateCollectionRequest = struct { - name: []const u8, - schema: ?[]const u8 = null, -}; - -fn handleDropCollection(request: *std.http.Server.Request, name: []const u8) !void { - bridge.dropCollection(name) catch |err| { - log.err("Failed to drop collection {s}: {}", .{ name, err }); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"LITH_ERR_NOT_IMPLEMENTED","message":"Collection drop not yet implemented in bridge"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - - request.respond("", .{ - .status = .no_content, - }) catch {}; -} - -// ============================================================================= -// Journal Handler -// ============================================================================= - -fn handleJournal( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - method: std.http.Method, -) !void { - if (method != .GET) { - try sendMethodNotAllowed(request); - return; - } - - // Parse query parameters from the URL target - // Expected: ?since=&limit= - const target = request.head.target; - var since: u64 = 0; - var limit: u32 = 100; - - if (std.mem.indexOf(u8, target, "?")) |q_idx| { - const query_string = target[q_idx + 1 ..]; - var param_iter = std.mem.splitScalar(u8, query_string, '&'); - while (param_iter.next()) |param| { - if (std.mem.startsWith(u8, param, "since=")) { - since = std.fmt.parseInt(u64, param["since=".len..], 10) catch 0; - } else if (std.mem.startsWith(u8, param, "limit=")) { - limit = std.fmt.parseInt(u32, param["limit=".len..], 10) catch 100; - } - } - } - - const entries = bridge.getJournal(since, limit) catch |err| { - log.err("Failed to get journal entries: {}", .{err}); - - const error_response = switch (err) { - error.NotInitialized => - \\{"error":"LITH_ERR_INTERNAL","message":"Database not initialized"} - , - error.JournalRenderFailed => - \\{"error":"LITH_ERR_IO_ERROR","message":"Failed to render journal entries"} - , - else => - \\{"error":"LITH_ERR_INTERNAL","message":"Internal server error"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - defer allocator.free(entries); - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.writeAll("{\"entries\":["); - for (entries, 0..) |entry, i| { - if (i > 0) try writer.writeByte(','); - try writer.print( - \\{{"seq":{d},"timestamp":"{s}","operation":"{s}" - , .{ entry.sequence, entry.timestamp, entry.operation }); - if (entry.collection) |col| { - try writer.print(",\"collection\":\"{s}\"", .{col}); - } - if (entry.actor) |act| { - try writer.print(",\"provenance\":{{\"actor\":\"{s}\"}}", .{act}); - } - try writer.writeByte('}'); - } - - // Compute hasMore: if we got exactly `limit` entries, there may be more - const has_more = entries.len == limit; - // nextSeq is the sequence after the last entry, or `since` if no entries - const next_seq = if (entries.len > 0) entries[entries.len - 1].sequence + 1 else since; - - try writer.print("],\"hasMore\":{s},\"nextSeq\":{d}}}", .{ - if (has_more) "true" else "false", - next_seq, - }); - - try sendJson(request, .ok, response_buffer.items); -} - -// ============================================================================= -// Normalize Handler -// ============================================================================= - -fn handleNormalize( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - method: std.http.Method, - endpoint: []const u8, -) !void { - if (method != .POST) { - try sendMethodNotAllowed(request); - return; - } - - if (std.mem.indexOf(u8, endpoint, "/discover")) |_| { - try handleDiscover(allocator, request); - } else if (std.mem.indexOf(u8, endpoint, "/analyze")) |_| { - try handleAnalyze(allocator, request); - } else { - try sendNotFound(request); - } -} - -fn handleDiscover(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - // Read request body to get collection name and optional sample_size - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 1024 * 1024); - defer allocator.free(body); - - const parsed = json.parseFromSlice(DiscoverRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - const sample_size = req.sample_size orelse 1000; - - const deps = bridge.discoverDependencies(req.collection, sample_size) catch |err| { - log.err("Failed to discover dependencies for {s}: {}", .{ req.collection, err }); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"LITH_ERR_NOT_IMPLEMENTED","message":"Dependency discovery not yet implemented in bridge"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - defer allocator.free(deps); - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print("{{\"collection\":\"{s}\",\"functionalDependencies\":[", .{req.collection}); - for (deps, 0..) |dep, i| { - if (i > 0) try writer.writeByte(','); - // Write determinant array - try writer.writeAll("{\"determinant\":["); - for (dep.determinant, 0..) |det, j| { - if (j > 0) try writer.writeByte(','); - try writer.print("\"{s}\"", .{det}); - } - // Classify confidence tier - const tier: []const u8 = if (dep.confidence >= 0.95) "high" else if (dep.confidence >= 0.8) "medium" else "low"; - try writer.print("],\"dependent\":\"{s}\",\"confidence\":{d:.2},\"tier\":\"{s}\"}}", .{ - dep.dependent, - dep.confidence, - tier, - }); - } - try writer.writeAll("],\"candidateKeys\":[]}"); - - try sendJson(request, .ok, response_buffer.items); -} - -const DiscoverRequest = struct { - collection: []const u8, - sample_size: ?u32 = null, -}; - -fn handleAnalyze(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - // Read request body to get collection name - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 1024 * 1024); - defer allocator.free(body); - - const parsed = json.parseFromSlice(AnalyzeRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - - const analysis = bridge.analyzeNormalForm(req.collection) catch |err| { - log.err("Failed to analyze normal form for {s}: {}", .{ req.collection, err }); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"LITH_ERR_NOT_IMPLEMENTED","message":"Normal form analysis not yet implemented in bridge"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print("{{\"collection\":\"{s}\",\"currentForm\":\"{s}\",\"violations\":[", .{ - req.collection, - analysis.current_form, - }); - - for (analysis.violations, 0..) |violation, i| { - if (i > 0) try writer.writeByte(','); - try writer.print("\"{s}\"", .{violation}); - } - - try writer.writeAll("],\"recommendations\":["); - for (analysis.suggestions, 0..) |suggestion, i| { - if (i > 0) try writer.writeByte(','); - try writer.print("\"{s}\"", .{suggestion}); - } - - try writer.writeAll("]}"); - - try sendJson(request, .ok, response_buffer.items); -} - -const AnalyzeRequest = struct { - collection: []const u8, -}; - -// ============================================================================= -// Migrate Handler -// ============================================================================= - -fn handleMigrate( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - method: std.http.Method, - endpoint: []const u8, -) !void { - if (method != .POST) { - try sendMethodNotAllowed(request); - return; - } - - if (std.mem.indexOf(u8, endpoint, "/start")) |_| { - try handleMigrationStart(allocator, request); - } else if (std.mem.indexOf(u8, endpoint, "/shadow")) |_| { - try handleMigrationAdvance(allocator, request, .start_shadow); - } else if (std.mem.indexOf(u8, endpoint, "/commit")) |_| { - try handleMigrationAdvance(allocator, request, .commit); - } else if (std.mem.indexOf(u8, endpoint, "/abort")) |_| { - try handleMigrationAdvance(allocator, request, .abort); - } else { - try sendNotFound(request); - } -} - -const MigrationStartRequest = struct { - collection: []const u8, - target_schema: ?[]const u8 = null, -}; - -const MigrationAdvanceRequest = struct { - id: []const u8, -}; - -fn handleMigrationStart(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - // Read request body - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 1024 * 1024); - defer allocator.free(body); - - const parsed = json.parseFromSlice(MigrationStartRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - const target_schema = req.target_schema orelse "{}"; - - const migration = bridge.startMigration(req.collection, target_schema) catch |err| { - log.err("Failed to start migration for {s}: {}", .{ req.collection, err }); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"LITH_ERR_NOT_IMPLEMENTED","message":"Migration not yet implemented in bridge"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - const phase_str = switch (migration.state) { - .announced => "announce", - .shadow_running => "shadow", - .shadow_complete => "shadow_complete", - .committed => "complete", - .aborted => "aborted", - }; - - try writer.print( - \\{{"id":"{s}","collection":"{s}","phase":"{s}","startedAt":"{s}","narrative":"Migration announced for collection {s}"}} - , .{ migration.id, migration.source_collection, phase_str, migration.created_at, migration.source_collection }); - - try sendJson(request, .ok, response_buffer.items); -} - -fn handleMigrationAdvance(allocator: std.mem.Allocator, request: *std.http.Server.Request, action: bridge.MigrationAction) !void { - // Read request body - var read_buf: [65536]u8 = undefined; - const body_reader = try request.readerExpectContinue(&read_buf); - const body = try body_reader.adaptToOldInterface().readAllAlloc(allocator, 1024 * 1024); - defer allocator.free(body); - - const parsed = json.parseFromSlice(MigrationAdvanceRequest, allocator, body, .{}) catch { - try sendBadRequest(request, "Invalid JSON in request body"); - return; - }; - defer parsed.deinit(); - - const req = parsed.value; - - bridge.advanceMigration(req.id, action) catch |err| { - log.err("Failed to advance migration {s}: {}", .{ req.id, err }); - - const error_response: []const u8 = switch (err) { - error.NotImplemented => - \\{"error":"LITH_ERR_NOT_IMPLEMENTED","message":"Migration advancement not yet implemented in bridge"} - , - }; - try sendJson(request, .internal_server_error, error_response); - return; - }; - - // Retrieve updated migration state - const migration = bridge.getMigration(req.id) catch |err| { - log.err("Failed to get migration {s} after advance: {}", .{ req.id, err }); - // The advance succeeded but we cannot read back the state; return minimal confirmation - const phase_str = switch (action) { - .start_shadow => "shadow", - .commit => "complete", - .abort => "aborted", - }; - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - try response_buffer.writer(allocator).print( - \\{{"id":"{s}","phase":"{s}","narrative":"Migration phase advanced"}} - , .{ req.id, phase_str }); - try sendJson(request, .ok, response_buffer.items); - return; - }; - - if (migration) |mig| { - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - const phase_str = switch (mig.state) { - .announced => "announce", - .shadow_running => "shadow", - .shadow_complete => "shadow_complete", - .committed => "complete", - .aborted => "aborted", - }; - - try writer.print( - \\{{"id":"{s}","collection":"{s}","phase":"{s}","startedAt":"{s}","narrative":"Migration phase: {s}"}} - , .{ mig.id, mig.source_collection, phase_str, mig.created_at, phase_str }); - - try sendJson(request, .ok, response_buffer.items); - } else { - try sendNotFound(request); - } -} - -// ============================================================================= -// Health & Metrics -// ============================================================================= - -fn handleHealth(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - const health = bridge.getHealth(); - - // Build response JSON - var response_buffer: std.ArrayList(u8) = .empty; - defer response_buffer.deinit(allocator); - const writer = response_buffer.writer(allocator); - - try writer.print( - \\{{"status":"{s}","version":"{s}","uptime":{d},"checks":{{"database":"{s}","journal":"{s}"}}}} - , .{ - health.status, - health.version, - health.uptime_seconds, - if (bridge.isInitialized()) "pass" else "fail", - if (bridge.isInitialized()) "pass" else "fail", - }); - - try sendJson(request, .ok, response_buffer.items); -} - -fn handleMetrics(allocator: std.mem.Allocator, request: *std.http.Server.Request) !void { - const prometheus_metrics = try metrics.getPrometheus(allocator); - defer allocator.free(prometheus_metrics); - - request.respond(prometheus_metrics, .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "text/plain; version=0.0.4" }, - }, - }) catch {}; -} - -// ============================================================================= -// Response Helpers -// ============================================================================= - -fn sendJson(request: *std.http.Server.Request, status: std.http.Status, body: []const u8) !void { - request.respond(body, .{ - .status = status, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -fn sendBadRequest(request: *std.http.Server.Request, message: []const u8) !void { - _ = message; // Message content not embedded in JSON to avoid injection; using static response - const body = - \\{"error":"bad_request","message":"Invalid request"} - ; - try sendJson(request, .bad_request, body); -} - -fn sendUnauthorized(request: *std.http.Server.Request) !void { - const body = - \\{"error":"unauthorized","message":"Authentication required"} - ; - try sendJson(request, .unauthorized, body); -} - -fn sendNotFound(request: *std.http.Server.Request) !void { - const body = - \\{"error":"not_found","message":"Resource not found"} - ; - try sendJson(request, .not_found, body); -} - -fn sendMethodNotAllowed(request: *std.http.Server.Request) !void { - const body = - \\{"error":"method_not_allowed","message":"Method not allowed for this endpoint"} - ; - request.respond(body, .{ - .status = .method_not_allowed, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -test "extract collection name" { - try std.testing.expectEqualStrings("articles", extractCollectionName("/collections/articles").?); - try std.testing.expectEqual(@as(?[]const u8, null), extractCollectionName("/collections")); - try std.testing.expectEqual(@as(?[]const u8, null), extractCollectionName("/collections/")); -} diff --git a/lithoglyph/api/src/router.zig b/lithoglyph/api/src/router.zig deleted file mode 100644 index c466fb91..00000000 --- a/lithoglyph/api/src/router.zig +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - HTTP Router - -const std = @import("std"); - -pub const Route = struct { - method: std.http.Method, - path: []const u8, - handler: *const fn ( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - params: std.StringHashMap([]const u8), - ) anyerror!void, -}; - -pub const Router = struct { - routes: std.ArrayList(Route), - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) Router { - return .{ - .routes = std.ArrayList(Route).init(allocator), - .allocator = allocator, - }; - } - - pub fn deinit(self: *Router) void { - self.routes.deinit(); - } - - pub fn addRoute(self: *Router, method: std.http.Method, path: []const u8, handler: anytype) !void { - try self.routes.append(.{ - .method = method, - .path = path, - .handler = handler, - }); - } - - pub fn get(self: *Router, path: []const u8, handler: anytype) !void { - try self.addRoute(.GET, path, handler); - } - - pub fn post(self: *Router, path: []const u8, handler: anytype) !void { - try self.addRoute(.POST, path, handler); - } - - pub fn delete(self: *Router, path: []const u8, handler: anytype) !void { - try self.addRoute(.DELETE, path, handler); - } - - pub fn match(self: *const Router, method: std.http.Method, path: []const u8) ?MatchResult { - for (self.routes.items) |route| { - if (route.method == method) { - if (matchPath(route.path, path)) |params| { - return .{ - .handler = route.handler, - .params = params, - }; - } - } - } - return null; - } -}; - -pub const MatchResult = struct { - handler: *const fn ( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - params: std.StringHashMap([]const u8), - ) anyerror!void, - params: std.StringHashMap([]const u8), -}; - -fn matchPath(pattern: []const u8, path: []const u8) ?std.StringHashMap([]const u8) { - var params = std.StringHashMap([]const u8).init(std.heap.page_allocator); - - var pattern_iter = std.mem.splitScalar(u8, pattern, '/'); - var path_iter = std.mem.splitScalar(u8, path, '/'); - - while (true) { - const pattern_part = pattern_iter.next(); - const path_part = path_iter.next(); - - if (pattern_part == null and path_part == null) { - return params; - } - - if (pattern_part == null or path_part == null) { - params.deinit(); - return null; - } - - const p = pattern_part.?; - const v = path_part.?; - - if (p.len > 0 and p[0] == ':') { - // Parameter - params.put(p[1..], v) catch { - params.deinit(); - return null; - }; - } else if (!std.mem.eql(u8, p, v)) { - params.deinit(); - return null; - } - } -} - -test "path matching" { - const router = Router.init(std.testing.allocator); - _ = router; - - // Basic path matching - if (matchPath("/v1/collections", "/v1/collections")) |params| { - defer params.deinit(); - try std.testing.expectEqual(@as(usize, 0), params.count()); - } else { - return error.TestFailed; - } - - // Parameter extraction - if (matchPath("/v1/collections/:name", "/v1/collections/articles")) |params| { - defer params.deinit(); - try std.testing.expectEqualStrings("articles", params.get("name").?); - } else { - return error.TestFailed; - } -} diff --git a/lithoglyph/api/src/websocket.zig b/lithoglyph/api/src/websocket.zig deleted file mode 100644 index 93a428f7..00000000 --- a/lithoglyph/api/src/websocket.zig +++ /dev/null @@ -1,332 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph API Server - WebSocket Handler -// -// WebSocket support for GraphQL subscriptions and real-time journal streaming - -const std = @import("std"); -const config = @import("config.zig"); -const bridge = @import("bridge_client.zig"); - -const log = std.log.scoped(.websocket); - -// WebSocket opcodes -const Opcode = enum(u4) { - continuation = 0, - text = 1, - binary = 2, - close = 8, - ping = 9, - pong = 10, -}; - -// Active subscriptions -pub const Subscription = struct { - id: []const u8, - subscription_type: SubscriptionType, - last_seq: u64, - allocator: std.mem.Allocator, - - pub fn deinit(self: *Subscription) void { - self.allocator.free(self.id); - } -}; - -pub const SubscriptionType = enum { - journal_stream, - collection_changes, - migration_progress, -}; - -// WebSocket connection state -pub const Connection = struct { - allocator: std.mem.Allocator, - subscriptions: std.StringHashMap(Subscription), - is_open: bool, - - pub fn init(allocator: std.mem.Allocator) Connection { - return .{ - .allocator = allocator, - .subscriptions = std.StringHashMap(Subscription).init(allocator), - .is_open = true, - }; - } - - pub fn deinit(self: *Connection) void { - var iter = self.subscriptions.valueIterator(); - while (iter.next()) |sub| { - sub.deinit(); - } - self.subscriptions.deinit(); - } - - pub fn addSubscription(self: *Connection, id: []const u8, sub_type: SubscriptionType) !void { - const id_copy = try self.allocator.dupe(u8, id); - try self.subscriptions.put(id_copy, .{ - .id = id_copy, - .subscription_type = sub_type, - .last_seq = 0, - .allocator = self.allocator, - }); - } - - pub fn removeSubscription(self: *Connection, id: []const u8) void { - if (self.subscriptions.fetchRemove(id)) |entry| { - var sub = entry.value; - sub.deinit(); - } - } -}; - -/// Check if a request is a WebSocket upgrade request -pub fn isUpgradeRequest(request: *std.http.Server.Request) bool { - const upgrade = getHeader(request, "upgrade") orelse return false; - return std.ascii.eqlIgnoreCase(upgrade, "websocket"); -} - -/// Handle WebSocket upgrade and connection -pub fn handleUpgrade( - allocator: std.mem.Allocator, - request: *std.http.Server.Request, - cfg: *const config.Config, -) !void { - _ = cfg; - - // Validate WebSocket upgrade request - const key = getHeader(request, "sec-websocket-key") orelse { - try sendBadRequest(request, "Missing Sec-WebSocket-Key header"); - return; - }; - - const version = getHeader(request, "sec-websocket-version") orelse "13"; - if (!std.mem.eql(u8, version, "13")) { - try sendBadRequest(request, "Unsupported WebSocket version"); - return; - } - - // Calculate accept key - const accept_key = try computeAcceptKey(allocator, key); - defer allocator.free(accept_key); - - // Send upgrade response - request.respond("", .{ - .status = .switching_protocols, - .extra_headers = &.{ - .{ .name = "upgrade", .value = "websocket" }, - .{ .name = "connection", .value = "Upgrade" }, - .{ .name = "sec-websocket-accept", .value = accept_key }, - .{ .name = "sec-websocket-protocol", .value = "graphql-ws" }, - }, - }) catch {}; - - log.info("WebSocket connection upgraded", .{}); - - // Note: After upgrade, the connection should be handled differently - // The HTTP server needs to hand off the raw socket to WebSocket handling - // This is a simplified implementation - full WebSocket handling would - // require access to the underlying socket for bidirectional communication -} - -/// Compute WebSocket accept key from client key -fn computeAcceptKey(allocator: std.mem.Allocator, key: []const u8) ![]const u8 { - const magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - - // Concatenate key + magic - var combined = try allocator.alloc(u8, key.len + magic.len); - defer allocator.free(combined); - @memcpy(combined[0..key.len], key); - @memcpy(combined[key.len..], magic); - - // SHA-1 hash - var hash: [20]u8 = undefined; - std.crypto.hash.Sha1.hash(combined, &hash, .{}); - - // Base64 encode - const encoded_len = std.base64.standard.Encoder.calcSize(20); - const encoded = try allocator.alloc(u8, encoded_len); - _ = std.base64.standard.Encoder.encode(encoded, &hash); - - return encoded; -} - -/// Encode a WebSocket frame -pub fn encodeFrame(allocator: std.mem.Allocator, opcode: Opcode, payload: []const u8) ![]const u8 { - // Calculate frame size - const header_size: usize = if (payload.len < 126) - 2 - else if (payload.len < 65536) - 4 - else - 10; - - var frame = try allocator.alloc(u8, header_size + payload.len); - - // First byte: FIN bit (1) + opcode - frame[0] = 0x80 | @intFromEnum(opcode); - - // Second byte: mask bit (0 for server) + payload length - if (payload.len < 126) { - frame[1] = @intCast(payload.len); - } else if (payload.len < 65536) { - frame[1] = 126; - std.mem.writeInt(u16, frame[2..4], @intCast(payload.len), .big); - } else { - frame[1] = 127; - std.mem.writeInt(u64, frame[2..10], payload.len, .big); - } - - // Payload - @memcpy(frame[header_size..], payload); - - return frame; -} - -/// Create a GraphQL subscription message (graphql-ws protocol) -pub fn createSubscriptionMessage( - allocator: std.mem.Allocator, - msg_type: []const u8, - id: ?[]const u8, - payload: ?[]const u8, -) ![]const u8 { - var buffer: std.ArrayList(u8) = .empty; - errdefer buffer.deinit(allocator); - const writer = buffer.writer(allocator); - - try writer.writeAll("{\"type\":\""); - try writer.writeAll(msg_type); - try writer.writeAll("\""); - - if (id) |i| { - try writer.writeAll(",\"id\":\""); - try writer.writeAll(i); - try writer.writeAll("\""); - } - - if (payload) |p| { - try writer.writeAll(",\"payload\":"); - try writer.writeAll(p); - } - - try writer.writeAll("}"); - - return try buffer.toOwnedSlice(allocator); -} - -/// Parse a GraphQL subscription message -pub const SubscriptionMessage = struct { - msg_type: []const u8, - id: ?[]const u8, - payload: ?[]const u8, -}; - -pub fn parseSubscriptionMessage(allocator: std.mem.Allocator, data: []const u8) !SubscriptionMessage { - const parsed = std.json.parseFromSlice(std.json.Value, allocator, data, .{}) catch { - return error.InvalidJson; - }; - defer parsed.deinit(); - - const root = parsed.value.object; - - const msg_type = if (root.get("type")) |t| switch (t) { - .string => |s| s, - else => return error.InvalidMessageType, - } else return error.MissingType; - - const id = if (root.get("id")) |i| switch (i) { - .string => |s| s, - else => null, - } else null; - - // For payload, we'd need to stringify it - simplified for now - const payload: ?[]const u8 = null; - - return .{ - .msg_type = msg_type, - .id = id, - .payload = payload, - }; -} - -/// Create a journal entry notification -pub fn createJournalNotification( - allocator: std.mem.Allocator, - sub_id: []const u8, - entry: bridge.JournalEntry, -) ![]const u8 { - var payload: std.ArrayList(u8) = .empty; - defer payload.deinit(allocator); - const writer = payload.writer(allocator); - - try writer.print( - \\{{"data":{{"journalStream":{{"seq":{d},"timestamp":"{s}","operation":"{s}" - , .{ entry.sequence, entry.timestamp, entry.operation }); - - if (entry.collection) |col| { - try writer.print(",\"collection\":\"{s}\"", .{col}); - } - - try writer.writeAll("}}}}"); - - return try createSubscriptionMessage(allocator, "next", sub_id, payload.items); -} - -// ============================================================================= -// Helper Functions -// ============================================================================= - -fn getHeader(request: *std.http.Server.Request, name: []const u8) ?[]const u8 { - var iter = request.iterateHeaders(); - while (iter.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, name)) { - return header.value; - } - } - return null; -} - -fn sendBadRequest(request: *std.http.Server.Request, message: []const u8) !void { - _ = message; - request.respond( - \\{"error":"bad_request"} - , .{ - .status = .bad_request, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - }, - }) catch {}; -} - -// ============================================================================= -// Tests -// ============================================================================= - -test "websocket accept key computation" { - const allocator = std.testing.allocator; - - // Example from RFC 6455 - const key = "dGhlIHNhbXBsZSBub25jZQ=="; - const accept = try computeAcceptKey(allocator, key); - defer allocator.free(accept); - - try std.testing.expectEqualStrings("s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", accept); -} - -test "websocket frame encoding" { - const allocator = std.testing.allocator; - - // Small payload - const frame = try encodeFrame(allocator, .text, "Hello"); - defer allocator.free(frame); - - try std.testing.expectEqual(@as(u8, 0x81), frame[0]); // FIN + text - try std.testing.expectEqual(@as(u8, 5), frame[1]); // Length - try std.testing.expectEqualStrings("Hello", frame[2..7]); -} - -test "subscription message creation" { - const allocator = std.testing.allocator; - - const msg = try createSubscriptionMessage(allocator, "connection_ack", null, null); - defer allocator.free(msg); - - try std.testing.expectEqualStrings("{\"type\":\"connection_ack\"}", msg); -} diff --git a/lithoglyph/beam/.editorconfig b/lithoglyph/beam/.editorconfig deleted file mode 100644 index fc6650ce..00000000 --- a/lithoglyph/beam/.editorconfig +++ /dev/null @@ -1,68 +0,0 @@ -# RSR-template-repo - Editor Configuration -# https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 2 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -trim_trailing_whitespace = false - -[*.adoc] -trim_trailing_whitespace = false - -[*.rs] -indent_size = 4 - -[*.ex] -indent_size = 2 - -[*.exs] -indent_size = 2 - -[*.zig] -indent_size = 4 - -[*.ada] -indent_size = 3 - -[*.adb] -indent_size = 3 - -[*.ads] -indent_size = 3 - -[*.hs] -indent_size = 2 - -[*.res] -indent_size = 2 - -[*.resi] -indent_size = 2 - -[*.ncl] -indent_size = 2 - -[*.rkt] -indent_size = 2 - -[*.scm] -indent_size = 2 - -[*.nix] -indent_size = 2 - -[Justfile] -indent_style = space -indent_size = 4 - -[justfile] -indent_style = space -indent_size = 4 diff --git a/lithoglyph/beam/.gitattributes b/lithoglyph/beam/.gitattributes deleted file mode 100644 index e860a85c..00000000 --- a/lithoglyph/beam/.gitattributes +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitattributes - -* text=auto eol=lf - -# Source -*.rs text eol=lf diff=rust -*.ex text eol=lf diff=elixir -*.exs text eol=lf diff=elixir -*.jl text eol=lf -*.res text eol=lf -*.resi text eol=lf -*.ada text eol=lf diff=ada -*.adb text eol=lf diff=ada -*.ads text eol=lf diff=ada -*.hs text eol=lf -*.chpl text eol=lf -*.scm text eol=lf -*.ncl text eol=lf -*.nix text eol=lf - -# Docs -*.md text eol=lf diff=markdown -*.adoc text eol=lf -*.txt text eol=lf - -# Data -*.json text eol=lf -*.yaml text eol=lf -*.yml text eol=lf -*.toml text eol=lf - -# Config -.gitignore text eol=lf -.gitattributes text eol=lf -justfile text eol=lf -Makefile text eol=lf -Containerfile text eol=lf - -# Scripts -*.sh text eol=lf - -# Binary -*.png binary -*.jpg binary -*.gif binary -*.pdf binary -*.woff2 binary -*.zip binary -*.gz binary - -# Lock files -Cargo.lock text eol=lf -diff -flake.lock text eol=lf -diff diff --git a/lithoglyph/beam/.gitignore b/lithoglyph/beam/.gitignore deleted file mode 100644 index cc9fc672..00000000 --- a/lithoglyph/beam/.gitignore +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitignore - -# OS & Editor -.DS_Store -Thumbs.db -*.swp -*.swo -*~ -.idea/ -.vscode/ - -# Build -/target/ -/_build/ -/build/ -/dist/ -/out/ - -# Dependencies -/node_modules/ -/vendor/ -/deps/ -/.elixir_ls/ - -# Rust -# Cargo.lock # Keep for binaries - -# Elixir -/cover/ -/doc/ -*.ez -erl_crash.dump - -# Julia -*.jl.cov -*.jl.mem -/Manifest.toml - -# ReScript -/lib/bs/ -/.bsb.lock - -# Python (SaltStack only) -__pycache__/ -*.py[cod] -.venv/ - -# Ada/SPARK -*.ali -/obj/ -/bin/ - -# Haskell -/.stack-work/ -/dist-newstyle/ - -# Chapel -*.chpl.tmp.* - -# Secrets -.env -.env.* -*.pem -*.key -secrets/ - -# Test/Coverage -/coverage/ -htmlcov/ - -# Logs -*.log -/logs/ - -# Temp -/tmp/ -*.tmp -*.bak -target/ -node_modules/ -_build/ -deps/ -.elixir_ls/ -.cache/ -build/ -dist/ diff --git a/lithoglyph/beam/.machine_readable/6a2/ECOSYSTEM.a2ml b/lithoglyph/beam/.machine_readable/6a2/ECOSYSTEM.a2ml deleted file mode 100644 index 2863e46e..00000000 --- a/lithoglyph/beam/.machine_readable/6a2/ECOSYSTEM.a2ml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# ECOSYSTEM.a2ml — Beam ecosystem position -[metadata] -version = "1.0.0" -last-updated = "2026-04-11" - -[project] -name = "Beam" -purpose = "Hyperpolymath ecosystem component" -role = "library" - -[position-in-ecosystem] -category = "" - -[related-projects] -projects = [ - # No related projects recorded -] diff --git a/lithoglyph/beam/.machine_readable/6a2/META.a2ml b/lithoglyph/beam/.machine_readable/6a2/META.a2ml deleted file mode 100644 index 9e0b940c..00000000 --- a/lithoglyph/beam/.machine_readable/6a2/META.a2ml +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# META.a2ml — Beam meta-level information -[metadata] -version = "1.0.0" -last-updated = "2026-04-11" - -[project-info] -license = "PMPL-1.0-or-later" -author = "Jonathan D.A. Jewell (hyperpolymath)" - -[architecture-decisions] -decisions = [ - # No ADRs recorded -] - -[development-practices] -versioning = "SemVer" -documentation = "AsciiDoc" -build-tool = "just" - -[maintenance-axes] -scoping-first = true -axis-1 = "must > intend > like" -axis-2 = "corrective > adaptive > perfective" -axis-3 = "systems > compliance > effects" diff --git a/lithoglyph/beam/.machine_readable/6a2/STATE.a2ml b/lithoglyph/beam/.machine_readable/6a2/STATE.a2ml deleted file mode 100644 index 977a93ae..00000000 --- a/lithoglyph/beam/.machine_readable/6a2/STATE.a2ml +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# STATE.a2ml — Beam project state -[metadata] -project = "lithoglyph-beam" -version = "0.1.0" -last-updated = "2025-01-17" -status = "active" -session = "converted from scheme — 2026-04-11" - -[project-context] -name = "Lithoglyph Beam" -purpose = """BEAM (Erlang/Gleam/Elixir) bindings for Lith""" -completion-percentage = 15 - -[position] -phase = "alpha" # design | implementation | testing | maintenance | archived -maturity = "experimental" # experimental | alpha | beta | production | lts - -[route-to-mvp] -milestones = [ - # No milestones recorded -] - -[blockers-and-issues] -issues = [ - # No blockers recorded -] - -[critical-next-actions] -actions = [ - # No actions recorded -] - -[maintenance-status] -last-run-utc = "2025-01-17T00:00:00Z" -last-result = "unknown" # unknown | pass | warn | fail diff --git a/lithoglyph/beam/ABI-FFI-README.md b/lithoglyph/beam/ABI-FFI-README.md deleted file mode 100644 index e6a32bbf..00000000 --- a/lithoglyph/beam/ABI-FFI-README.md +++ /dev/null @@ -1,385 +0,0 @@ -{{~ Aditionally delete this line and fill out the template below ~}} - -# {{PROJECT}} ABI/FFI Documentation - -## Overview - -This library follows the **Hyperpolymath RSR Standard** for ABI and FFI design: - -- **ABI (Application Binary Interface)** defined in **Idris2** with formal proofs -- **FFI (Foreign Function Interface)** implemented in **Zig** for C compatibility -- **Generated C headers** bridge Idris2 ABI to Zig FFI -- **Any language** can call through standard C ABI - -## Architecture - -``` -┌─────────────────────────────────────────────┐ -│ ABI Definitions (Idris2) │ -│ src/abi/ │ -│ - Types.idr (Type definitions) │ -│ - Layout.idr (Memory layout proofs) │ -│ - Foreign.idr (FFI declarations) │ -└─────────────────┬───────────────────────────┘ - │ - │ generates (at compile time) - ▼ -┌─────────────────────────────────────────────┐ -│ C Headers (auto-generated) │ -│ generated/abi/{{project}}.h │ -└─────────────────┬───────────────────────────┘ - │ - │ imported by - ▼ -┌─────────────────────────────────────────────┐ -│ FFI Implementation (Zig) │ -│ ffi/zig/src/main.zig │ -│ - Implements C-compatible functions │ -│ - Zero-cost abstractions │ -│ - Memory-safe by default │ -└─────────────────┬───────────────────────────┘ - │ - │ compiled to lib{{project}}.so/.a - ▼ -┌─────────────────────────────────────────────┐ -│ Any Language via C ABI │ -│ - Rust, ReScript, Julia, Python, etc. │ -└─────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -{{project}}/ -├── src/ -│ ├── abi/ # ABI definitions (Idris2) -│ │ ├── Types.idr # Core type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI function declarations -│ └── lib/ # Core library (any language) -│ -├── ffi/ -│ └── zig/ # FFI implementation (Zig) -│ ├── build.zig # Build configuration -│ ├── build.zig.zon # Dependencies -│ ├── src/ -│ │ └── main.zig # C-compatible FFI implementation -│ ├── test/ -│ │ └── integration_test.zig -│ └── include/ -│ └── {{project}}.h # C header (optional, can be generated) -│ -├── generated/ # Auto-generated files -│ └── abi/ -│ └── {{project}}.h # Generated from Idris2 ABI -│ -└── bindings/ # Language-specific wrappers (optional) - ├── rust/ - ├── rescript/ - └── julia/ -``` - -## Why Idris2 for ABI? - -### 1. **Formal Verification** - -Idris2's dependent types allow proving properties about the ABI at compile-time: - -```idris --- Prove struct size is correct -public export -exampleStructSize : HasSize ExampleStruct 16 - --- Prove field alignment is correct -public export -fieldAligned : Divides 8 (offsetOf ExampleStruct.field) - --- Prove ABI is platform-compatible -public export -abiCompatible : Compatible (ABI 1) (ABI 2) -``` - -### 2. **Type Safety** - -Encode invariants that C/Zig cannot express: - -```idris --- Non-null pointer guaranteed at type level -data Handle : Type where - MkHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> Handle - --- Array with length proof -data Buffer : (n : Nat) -> Type where - MkBuffer : Vect n Byte -> Buffer n -``` - -### 3. **Platform Abstraction** - -Platform-specific types with compile-time selection: - -```idris -CInt : Platform -> Type -CInt Linux = Bits32 -CInt Windows = Bits32 - -CSize : Platform -> Type -CSize Linux = Bits64 -CSize Windows = Bits64 -``` - -### 4. **Safe Evolution** - -Prove that new ABI versions are backward-compatible: - -```idris --- Compiler enforces compatibility -abiUpgrade : ABI 1 -> ABI 2 -abiUpgrade old = MkABI2 { - -- Must preserve all v1 fields - v1_compat = old, - -- Can add new fields - new_features = defaults -} -``` - -## Why Zig for FFI? - -### 1. **C ABI Compatibility** - -Zig exports C-compatible functions naturally: - -```zig -export fn library_function(param: i32) i32 { - return param * 2; -} -``` - -### 2. **Memory Safety** - -Compile-time safety without runtime overhead: - -```zig -// Null check enforced at compile time -const handle = init() orelse return error.InitFailed; -defer free(handle); -``` - -### 3. **Cross-Compilation** - -Built-in cross-compilation to any platform: - -```bash -zig build -Dtarget=x86_64-linux -zig build -Dtarget=aarch64-macos -zig build -Dtarget=x86_64-windows -``` - -### 4. **Zero Dependencies** - -No runtime, no libc required (unless explicitly needed): - -```zig -// Minimal binary size -pub const lib = @import("std"); -// Only includes what you use -``` - -## Building - -### Build FFI Library - -```bash -cd ffi/zig -zig build # Build debug -zig build -Doptimize=ReleaseFast # Build optimized -zig build test # Run tests -``` - -### Generate C Header from Idris2 ABI - -```bash -cd src/abi -idris2 --cg c-header Types.idr -o ../../generated/abi/{{project}}.h -``` - -### Cross-Compile - -```bash -cd ffi/zig - -# Linux x86_64 -zig build -Dtarget=x86_64-linux - -# macOS ARM64 -zig build -Dtarget=aarch64-macos - -# Windows x86_64 -zig build -Dtarget=x86_64-windows -``` - -## Usage - -### From C - -```c -#include "{{project}}.h" - -int main() { - void* handle = {{project}}_init(); - if (!handle) return 1; - - int result = {{project}}_process(handle, 42); - if (result != 0) { - const char* err = {{project}}_last_error(); - fprintf(stderr, "Error: %s\n", err); - } - - {{project}}_free(handle); - return 0; -} -``` - -Compile with: -```bash -gcc -o example example.c -l{{project}} -L./zig-out/lib -``` - -### From Idris2 - -```idris -import {{PROJECT}}.ABI.Foreign - -main : IO () -main = do - Just handle <- init - | Nothing => putStrLn "Failed to initialize" - - Right result <- process handle 42 - | Left err => putStrLn $ "Error: " ++ errorDescription err - - free handle - putStrLn "Success" -``` - -### From Rust - -```rust -#[link(name = "{{project}}")] -extern "C" { - fn {{project}}_init() -> *mut std::ffi::c_void; - fn {{project}}_free(handle: *mut std::ffi::c_void); - fn {{project}}_process(handle: *mut std::ffi::c_void, input: u32) -> i32; -} - -fn main() { - unsafe { - let handle = {{project}}_init(); - assert!(!handle.is_null()); - - let result = {{project}}_process(handle, 42); - assert_eq!(result, 0); - - {{project}}_free(handle); - } -} -``` - -### From Julia - -```julia -const lib{{project}} = "lib{{project}}" - -function init() - handle = ccall((:{{project}}_init, lib{{project}}), Ptr{Cvoid}, ()) - handle == C_NULL && error("Failed to initialize") - handle -end - -function process(handle, input) - result = ccall((:{{project}}_process, lib{{project}}), Cint, (Ptr{Cvoid}, UInt32), handle, input) - result -end - -function cleanup(handle) - ccall((:{{project}}_free, lib{{project}}), Cvoid, (Ptr{Cvoid},), handle) -end - -# Usage -handle = init() -try - result = process(handle, 42) - println("Result: $result") -finally - cleanup(handle) -end -``` - -## Testing - -### Unit Tests (Zig) - -```bash -cd ffi/zig -zig build test -``` - -### Integration Tests - -```bash -cd ffi/zig -zig build test-integration -``` - -### ABI Verification (Idris2) - -```idris --- Compile-time verification -%runElab verifyABI - --- Runtime checks -main : IO () -main = do - verifyLayoutsCorrect - verifyAlignmentsCorrect - putStrLn "ABI verification passed" -``` - -## Contributing - -When modifying the ABI/FFI: - -1. **Update ABI first** (`src/abi/*.idr`) - - Modify type definitions - - Update proofs - - Ensure backward compatibility - -2. **Generate C header** - ```bash - idris2 --cg c-header src/abi/Types.idr -o generated/abi/{{project}}.h - ``` - -3. **Update FFI implementation** (`ffi/zig/src/main.zig`) - - Implement new functions - - Match ABI types exactly - -4. **Add tests** - - Unit tests in Zig - - Integration tests - - ABI verification tests - -5. **Update documentation** - - Function signatures - - Usage examples - - Migration guide (if breaking changes) - -## License - -PMPL-1.0-or-later - -## See Also - -- [Idris2 Documentation](https://idris2.readthedocs.io) -- [Zig Documentation](https://ziglang.org/documentation/master/) -- [Rhodium Standard Repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [FFI Migration Guide](../ffi-migration-guide.md) -- [ABI Migration Guide](../abi-migration-guide.md) diff --git a/lithoglyph/beam/BUILD-STATUS.md b/lithoglyph/beam/BUILD-STATUS.md deleted file mode 100644 index 4f878512..00000000 --- a/lithoglyph/beam/BUILD-STATUS.md +++ /dev/null @@ -1,178 +0,0 @@ -# Lith-BEAM Build Status - -**Date:** 2026-02-04 -**Session:** M10 Day 3 - Rustler Migration Complete -**Status:** ✅ 100% COMPLETE - All tests passing - -## What's Complete ✅ - -### 1. Rustler NIF Implementation - WORKING -- **Technology:** Rust + Rustler 0.35 -- **Location:** `native_rust/src/lib.rs` -- **Status:** All 9 NIF functions working -- **Test Results:** All tests passing (see below) - -### 2. BEAM API Wrapper -- **File:** `src/lith_nif.erl` -- **Purpose:** Erlang wrapper module for NIF loading -- **Status:** Working correctly - -### 3. Gleam Client - Already Complete -- High-level API with transactions -- Error handling -- CBOR binary handling - -### 4. Test Infrastructure - Working -- **File:** `test_rust.erl` -- **Tests:** All 9 NIF functions end-to-end -- **Status:** All tests passing ✓ - -## Test Results ✅ - -``` -=== Lithoglyph Rust NIF Test === -Test 1: Calling version()... - ✓ Version: {1,0,0} - -Test 2: Opening database... - ✓ Database opened - -Test 3: Beginning transaction... - ✓ Transaction started - -Test 4: Applying operation... - ✓ Operation applied, block ID: [0,0,0,0,0,0,0,1] - -Test 5: Committing transaction... - ✓ Transaction committed - -Test 6: Getting schema... - ✓ Schema: " " (CBOR empty map 0xa0) - -Test 7: Getting journal... - ✓ Journal: [128] (CBOR empty array 0x80) - -Test 8: Closing database... - ✓ Database closed - -=== All tests passed! === -``` - -## Implementation Details - -### NIF Functions - -| Function | Return Type | Status | Notes | -|----------|-------------|--------|-------| -| `version()` | `(i32, i32, i32)` | ✓ | Returns (1, 0, 0) | -| `db_open(path)` | `ResourceArc` | ✓ | Creates database handle | -| `db_close(db)` | `Atom` | ✓ | Returns `ok` | -| `txn_begin(db, mode)` | `Result, Atom>` | ✓ | ReadOnly/ReadWrite modes | -| `txn_commit(txn)` | `Atom` | ✓ | Returns `ok` | -| `txn_abort(txn)` | `Atom` | ✓ | Returns `ok` | -| `apply(txn, cbor)` | `Result, Atom>` | ✓ | Validates CBOR, returns block ID | -| `schema(db)` | `Vec` | ✓ | Returns CBOR empty map | -| `journal(db, since)` | `Vec` | ✓ | Returns CBOR empty array | - -### M10 PoC Stubs - -All functions use stub implementations for M10 testing: -- **version()**: Returns v1.0.0 -- **db_open()**: Creates dummy DbHandle (0xDEADBEEF marker) -- **txn_begin()**: Creates TxnHandle with mode -- **apply()**: Validates CBOR major type 5 (map), returns block_id = 1 -- **schema()**: Returns CBOR empty map (0xa0) -- **journal()**: Returns CBOR empty array (0x80) -- **commit/abort/close**: Return ok atom - -### CBOR Validation - -The `apply()` function validates CBOR input: -- Rejects empty or >1MB binaries -- Checks first byte major type is 5 (map) -- Returns `parse_failed` atom on invalid input - -## Build Instructions - -```bash -# Build Rust NIF -cd native_rust -cargo build --release - -# Copy to priv directory -cp target/release/liblith_nif.so ../priv/lith_nif.so - -# Compile Erlang wrapper -erlc -o ebin src/lith_nif.erl - -# Run tests -./test_rust.erl -``` - -## Migration Notes: Zig → Rustler - -**Decision:** Migrated from Zig to Rustler due to persistent segfault during NIF loading. - -**Root Cause (Zig):** -- Deep ABI incompatibility between Zig struct layout and Erlang expectations -- Inline functions required C shim layer -- Resource type system difficult to bridge - -**Why Rustler:** -- ✅ Proven BEAM compatibility -- ✅ Production-ready (used by many Elixir projects) -- ✅ Clean resource management -- ✅ Excellent error handling -- ✅ No ABI mismatch issues - -**Migration Time:** ~2 hours (as estimated) - -## Performance Targets (M10 PoC) - -| Operation | Current | Notes | -|-----------|---------|-------| -| `version()` | < 1μs | Direct return | -| `db_open()` | < 10μs | Creates resource | -| `txn_begin()` | < 5μs | Creates resource | -| `apply()` | < 50μs | CBOR validation only | -| `txn_commit()` | < 5μs | Returns atom | - -*Note: M11 will add actual Lithoglyph integration with gforth subprocess* - -## Next Steps - FormBase Integration - -Now that the NIF works, proceed to: - -### Priority 2: FormBase Testing (1-2 hours) -1. Test with actual Gleam FormBase client -2. Verify transaction flow works end-to-end -3. Test CBOR encoding/decoding roundtrip -4. Validate error handling - -### Priority 3: M11 HTTP API (2-3 hours) -1. Define API endpoints for Lith-Geo and Lith-Analytics -2. Choose framework (Phoenix/Plug) -3. Implement HTTP wrapper for Lithoglyph -4. Document API specification - -## Summary - -**Lith-BEAM is 100% complete for M10:** -- ✅ All 9 NIF functions implemented -- ✅ All tests passing -- ✅ CBOR validation working -- ✅ Resource management correct -- ✅ Ready for FormBase integration -- ✅ Rustler provides production-ready foundation - -**Time Investment:** -- Zig attempt: ~4 hours (learning + debugging) -- Rustler migration: ~2 hours -- **Total:** ~6 hours for complete working NIF - -**Result:** Production-ready BEAM bridge to Lithoglyph with proven reliability. - ---- - -**Session Date:** 2026-02-04 -**Completed By:** M10 Day 3 - Rustler migration successful diff --git a/lithoglyph/beam/EXTENDED-SESSION-SUMMARY.md b/lithoglyph/beam/EXTENDED-SESSION-SUMMARY.md deleted file mode 100644 index 202a056d..00000000 --- a/lithoglyph/beam/EXTENDED-SESSION-SUMMARY.md +++ /dev/null @@ -1,335 +0,0 @@ -# Extended Session Summary - M10 Day 3 + M11 Core API - -**Date:** 2026-02-04 -**Duration:** ~10 hours -**Status:** ALL DELIVERABLES COMPLETE ✅ - -## Executive Summary - -Completed all three M10 Day 3 priorities PLUS implemented M11 Core HTTP API. Lithoglyph ecosystem now has: -1. ✅ Working Rustler NIF (Lith-BEAM) -2. ✅ Gleam client integration (FormBase) -3. ✅ HTTP API specification + implementation (M11) -4. ✅ Security requirements documented - -## Session Breakdown - -### Part 1: M10 Day 3 (7 hours) - -#### ✅ Priority 1: Lith-BEAM Rustler Migration (2 hours) -**Problem:** Zig NIF had persistent segfault during `erlang:load_nif/2` -**Solution:** Migrated to Rustler (proven BEAM compatibility) -**Result:** All 9 NIF functions working, 8/8 tests passing - -**Technology:** Rust + Rustler 0.35 -**Files:** `native_rust/src/lib.rs`, `native_rust/Cargo.toml`, `src/lith_nif.erl` - -#### ✅ Priority 2: FormBase Integration Testing (3 hours) -**Goal:** Test Lith-BEAM NIF with Gleam FormBase client -**Result:** Gleam client successfully calls NIF, all tests passing - -**Technology:** Gleam + Erlang FFI + Rustler -**Files:** -- `formbase/server/src/lith/nif_ffi.gleam` - FFI declarations -- `formbase/server/src/lith/client.gleam` - High-level client -- `formbase/server/test_lith_nif.erl` - Integration tests - -**Challenges:** -- Gleam type system vs Erlang dynamic types -- `dynamic.Dynamic` handling for result tuples -- Unsafe coercion for opaque resource types - -#### ✅ Priority 3: M11 HTTP API Specification (2 hours) -**Goal:** Design REST API for Lith-Geo and Lith-Analytics -**Result:** Complete API specification (1900+ lines) - -**Endpoints Specified:** -- Core: version, database, transaction, operations, schema, journal -- Geo: insert, query, provenance -- Analytics: timeseries insert, query, aggregation -- WebSocket: real-time journal subscriptions - -**Documentation:** `M11-HTTP-API-SPEC.md` - -#### ✅ Bonus: Security Requirements (1 hour) -**Goal:** Capture comprehensive security requirements -**Result:** Post-quantum crypto roadmap, algorithm specs, implementation plan - -**Key Standards:** -- Password: Argon2id (512 MiB, 8 iter, 4 lanes) -- Hashing: SHAKE3-512 (FIPS 202) -- PQ Signatures: Dilithium5-AES (ML-DSA-87) -- PQ Key Exchange: Kyber-1024 (ML-KEM-1024) -- Symmetric: XChaCha20-Poly1305 (256-bit) -- Network: QUIC + HTTP/3 + IPv6 - -**Documentation:** `SECURITY-REQUIREMENTS.scm` - -### Part 2: M11 Core API Implementation (3 hours) - -#### ✅ Phoenix Project Setup (30 minutes) -- Created Phoenix 1.7 project (no Ecto, no HTML) -- Copied Rustler NIF from lithoglyph-beam -- Compiled NIF for Phoenix environment -- Set up correct directory structure - -#### ✅ Elixir Wrapper & Client (1 hour) -- Created `LithNif` module (Elixir → Erlang bridge) -- Created `Lithoglyph` high-level client API -- Implemented `with_transaction` helper -- Added proper error handling - -**Files:** -- `lib/lith_nif.ex` - Elixir NIF wrapper -- `lib/lith_http/lith.ex` - High-level client - -#### ✅ HTTP API Controllers (1 hour) -- Implemented `ApiController` with all 9 endpoints -- JSON request/response handling -- Base64 CBOR encoding/decoding -- Error responses with proper HTTP status codes -- Process dictionary for handle storage (PoC) - -**Files:** -- `lib/lith_http_web/controllers/api_controller.ex` -- `lib/lith_http_web/router.ex` - -#### ✅ Testing & Verification (30 minutes) -- Created Elixir test script (`test_api.exs`) -- Created HTTP test script (`test_http_api.sh`) -- All tests passing ✓ - -## Total Accomplishments - -### Repositories Modified - -1. **lithoglyph-beam** (Lith-BEAM) - - ✅ Rustler NIF implementation - - ✅ M10 PoC complete - - ✅ Documentation updated - -2. **formbase** (FormBase Server) - - ✅ Gleam client integration - - ✅ NIF FFI wrapper - - ✅ Integration tests - -3. **lith_http** (NEW - M11 HTTP API) - - ✅ Phoenix application - - ✅ Core HTTP endpoints - - ✅ Rustler NIF integration - -### Files Created/Modified - -**Total:** 30+ files -**Lines of Code:** ~3500 -**Test Scripts:** 5 -**Documentation:** 6 files - -### Tests Passing - -| Test Suite | Status | Count | -|------------|--------|-------| -| Lith-BEAM NIF | ✅ | 8/8 | -| FormBase Integration | ✅ | 8/8 | -| M11 Elixir API | ✅ | 6/6 | -| **Total** | **✅** | **22/22** | - -## Key Technical Achievements - -### 1. Zig → Rustler Migration Success -**Problem:** Deep ABI incompatibility causing segfaults -**Solution:** Rustler's proven BEAM integration -**Impact:** Saved ~10 hours of debugging, production-ready foundation - -### 2. Gleam FFI Mastery -**Challenge:** Gleam type system + Erlang dynamic types -**Solution:** `unsafe_coerce` for opaque resources, proper dynamic handling -**Impact:** Clean Gleam API wrapping Erlang NIF - -### 3. Phoenix NIF Integration -**Challenge:** NIF module naming, priv directory structure -**Solution:** Erlang `:lith_nif` module, correct build paths -**Impact:** Seamless NIF access from Phoenix controllers - -### 4. RESTful Design -**Achievement:** Clean, idiomatic REST API -**Features:** JSON, Base64 CBOR, proper status codes, error handling -**Impact:** Production-ready HTTP interface to Lithoglyph - -## Performance Metrics - -### NIF Operations (M10 PoC) -| Operation | Time | Notes | -|-----------|------|-------| -| version() | <1μs | Direct return | -| db_open() | <10μs | Resource creation | -| txn_begin() | <5μs | Resource creation | -| apply() | <50μs | CBOR validation | -| txn_commit() | <5μs | Atom return | - -### HTTP API Overhead -| Endpoint | Total | NIF | HTTP Overhead | -|----------|-------|-----|---------------| -| GET /version | ~500μs | <1μs | ~499μs | -| POST /databases | ~1ms | <10μs | ~990μs | -| POST /operations | ~1.5ms | <50μs | ~1.45ms | - -HTTP overhead: JSON encoding + Base64 + routing - -## Timeline - -| Time | Activity | Duration | -|------|----------|----------| -| 14:00-16:00 | Lith-BEAM Rustler migration | 2h | -| 16:00-19:00 | FormBase Gleam integration | 3h | -| 19:00-21:00 | M11 API specification | 2h | -| 21:00-21:30 | Security requirements | 0.5h | -| 21:30-22:00 | Phoenix setup | 0.5h | -| 22:00-23:00 | Elixir wrapper & client | 1h | -| 23:00-00:00 | HTTP controllers | 1h | -| 00:00-00:30 | Testing & verification | 0.5h | -| **Total** | | **~10.5h** | - -## M11 HTTP API Status - -### ✅ Implemented (Core API) -- Version endpoint -- Database management (create, close) -- Transaction management (begin, commit, abort) -- Operation application (CBOR) -- Schema retrieval -- Journal retrieval - -### ⏳ Remaining Work -- Lith-Geo endpoints (2-3 hours) -- Lith-Analytics endpoints (2-3 hours) -- WebSocket subscriptions (1-2 hours) -- Authentication (JWT) (1 hour) -- Rate limiting (1 hour) -- Metrics/monitoring (1 hour) - -**Total Remaining:** 8-12 hours - -## How to Use - -### Start Lithoglyph HTTP API Server -```bash -cd ~/Documents/hyperpolymath-repos/lith_http -mix phx.server -``` - -Server runs on `http://localhost:4000` - -### Test the API -```bash -# Elixir tests -mix run test_api.exs - -# HTTP tests (in another terminal) -./test_http_api.sh -``` - -### Example HTTP Request -```bash -# Get version -curl http://localhost:4000/api/v1/version - -# Create database -curl -X POST http://localhost:4000/api/v1/databases \ - -H "Content-Type: application/json" \ - -d '{"path": "/tmp/mydb"}' - -# Begin transaction -curl -X POST http://localhost:4000/api/v1/databases/db_abc123/transactions \ - -H "Content-Type: application/json" \ - -d '{"mode": "read_write"}' -``` - -## Next Session Priorities - -### 1. Lith-Geo Implementation (2-3 hours) -- GeoJSON parsing -- Spatial query implementation -- Bounding box queries -- Provenance tracking - -### 2. Lith-Analytics Implementation (2-3 hours) -- Time-series data model -- Aggregation queries (avg, min, max, sum) -- Interval-based grouping -- Provenance summaries - -### 3. WebSocket Subscriptions (1-2 hours) -- Phoenix Channel setup -- Journal subscription -- Real-time updates -- Connection management - -### 4. Production Hardening (2-3 hours) -- JWT authentication -- Rate limiting (Redis) -- Request logging -- Prometheus metrics -- OpenAPI documentation - -## Lessons Learned - -### 1. Rustler > Zig for BEAM NIFs -When in doubt, use battle-tested libraries. Rustler saved 4+ hours of ABI debugging. - -### 2. Specification Before Implementation -The 2 hours spent on M11 API specification made implementation trivial. Clear design prevents costly refactoring. - -### 3. Type Systems Catch Bugs -Gleam's type system caught multiple errors at compile-time that would have been runtime bugs. - -### 4. Phoenix is Excellent for APIs -Phoenix's controller structure, JSON handling, and routing made HTTP API implementation smooth. - -### 5. Documentation Pays Off -Comprehensive documentation (6 files, 3000+ lines) makes onboarding and debugging much easier. - -## Repository Status - -| Repo | Status | Ready for M12 | -|------|--------|---------------| -| **lithoglyph** (core-forth) | ✅ Complete | ✅ (C ABI built) | -| **lithoglyph-beam** | ✅ Complete | ✅ (Rustler NIF) | -| **formbase** | ✅ Complete | ✅ (Gleam client) | -| **lith_http** | ⚡ Core API | ⏳ (Geo/Analytics pending) | -| lithoglyph-geo | Spec only | ⏳ | -| lithoglyph-analytics | Spec only | ⏳ | -| lithoglyph-debugger | Not started | ⏳ | -| lithoglyph-studio | Not started | ⏳ | - -## Conclusion - -**Massive productivity session:** -- ✅ M10 Day 3: All 3 priorities complete -- ✅ M11 Core API: Implemented and tested -- ✅ Security: Comprehensive requirements documented -- ✅ Tests: 22/22 passing - -**Lithoglyph ecosystem status:** -- Production-ready NIF (Rustler) -- Working Gleam client -- Operational HTTP API -- Clear security roadmap - -**Impact:** -- M10 milestone: COMPLETE -- M11 milestone: 60% complete (core API done) -- M12 foundation: Fully prepared - -**Next step:** Implement Geo/Analytics endpoints to complete M11. - ---- - -**Session Date:** 2026-02-04 -**Total Time:** ~10.5 hours -**Total Commits:** 25+ -**Lines of Code:** ~3500 -**Files Created:** 30+ -**Tests Passing:** 22/22 ✓ -**Coffee Consumed:** ☕☕☕☕☕☕ -**Status:** 🎉 OUTSTANDING SUCCESS 🎉 diff --git a/lithoglyph/beam/LICENSE b/lithoglyph/beam/LICENSE deleted file mode 100644 index ec540b34..00000000 --- a/lithoglyph/beam/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: MPL-2.0 -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/lithoglyph/beam/LITH-BEAM-INTEGRATION.md b/lithoglyph/beam/LITH-BEAM-INTEGRATION.md deleted file mode 100644 index ceb433b0..00000000 --- a/lithoglyph/beam/LITH-BEAM-INTEGRATION.md +++ /dev/null @@ -1,373 +0,0 @@ - - -# Lith-BEAM Integration - READY FOR TESTING ✅ - -**Integration Date:** 2026-02-04 -**Status:** NIF implementation complete, ready for build and testing -**Blocks:** FormBase (now unblocked!) - -## Summary - -**Lith-BEAM** now has a complete NIF implementation connecting BEAM (Erlang/Gleam/Elixir) to Lithoglyph via the Lith C ABI created in M10 Day 3. - -## Architecture - -``` -Gleam Application (FormBase, etc.) - ↓ FFI calls - Gleam Client API (src/lith_beam/client.gleam) - ↓ NIF calls - Erlang NIF Module (native/src/lith_nif.erl) - ↓ Native calls - Zig NIF Implementation (native/src/lith_nif.zig) - ↓ C ABI calls - Lith C ABI (lith/database/core-forth/ffi/zig/src/abi.zig) - ↓ Subprocess (M10 PoC) - gforth Runtime - ↓ - Persistence Layer → Block Storage -``` - -## Components - -### 1. Gleam Client API (Existing ✓) -**File:** `src/lith_beam/client.gleam` -**Status:** Complete - -**Features:** -- Opaque types: `Connection`, `Transaction` -- Transaction modes: `ReadOnly`, `ReadWrite` -- Error handling: `LithoglyphError` with detailed error types -- High-level API: `with_transaction` (automatic commit/abort) - -**Public Functions:** -- `version() -> #(Int, Int, Int)` -- `connect(path: String) -> LithoglyphResult(Connection)` -- `disconnect(conn: Connection) -> LithoglyphResult(Nil)` -- `begin_transaction(conn, mode) -> LithoglyphResult(Transaction)` -- `commit(txn) -> LithoglyphResult(Nil)` -- `abort(txn) -> LithoglyphResult(Nil)` -- `apply_operation(txn, operation: BitArray) -> LithoglyphResult(#(BitArray, Option(BitArray)))` -- `get_schema(conn) -> LithoglyphResult(BitArray)` -- `get_journal(conn, since: Int) -> LithoglyphResult(BitArray)` - -### 2. Erlang NIF Module (Existing ✓) -**File:** `native/src/lith_nif.erl` -**Status:** Complete - -**Exports:** -- `version/0 -> {Major, Minor, Patch}` -- `db_open/1 -> {ok, DbRef} | {error, Reason}` -- `db_close/1 -> ok | {error, Reason}` -- `txn_begin/2 -> {ok, TxnRef} | {error, Reason}` -- `txn_commit/1 -> ok | {error, Reason}` -- `txn_abort/1 -> ok` -- `apply/2 -> {ok, ResultCbor} | {ok, ResultCbor, ProvenanceCbor} | {error, Reason}` -- `schema/1 -> {ok, SchemaCbor} | {error, Reason}` -- `journal/2 -> {ok, JournalCbor} | {error, Reason}` - -### 3. Zig NIF Implementation (NEW ✅) -**File:** `native/src/lith_nif.zig` -**Status:** Complete, ready for testing - -**Features:** -- Full NIF function implementations (9 functions) -- Resource management (DbHandle, TxnHandle) -- CBOR data handling via Lith C ABI -- Error handling with Erlang atoms -- Memory-safe with proper cleanup - -**Functions Implemented:** -- ✅ `version` - Returns Lithoglyph version (1, 0, 0) -- ✅ `db_open` - Creates DbHandle, calls `lithoglyph_init()` -- ✅ `db_close` - Cleanup DbHandle, calls `lithoglyph_cleanup()` -- ✅ `txn_begin` - Creates TxnHandle with mode -- ✅ `txn_commit` - Commits transaction (stub for M10) -- ✅ `txn_abort` - Aborts transaction -- ✅ `apply` - Parses CBOR, validates, persists via Lith C ABI -- ✅ `schema` - Returns empty CBOR map (stub for M10) -- ✅ `journal` - Returns empty CBOR array (stub for M10) - -**Lith C ABI Integration:** -```zig -const lithoglyph = struct { - extern fn lithoglyph_init() ?*anyopaque; - extern fn lithoglyph_cleanup(handle: ?*anyopaque) void; - extern fn lithoglyph_parse_cbor(handle: ?*anyopaque, cbor_data: [*]const u8, cbor_len: usize) ?*anyopaque; - extern fn lithoglyph_validate(token: ?*anyopaque) c_int; - extern fn lithoglyph_persist(handle: ?*anyopaque, token: ?*anyopaque) u64; - extern fn lithoglyph_load(handle: ?*anyopaque, block_id: u64) ?*anyopaque; -}; -``` - -### 4. BEAM API Helper (NEW ✅) -**File:** `native/src/beam.zig` -**Status:** Complete - -**Purpose:** Zig-friendly wrappers for Erlang NIF C API - -**Provided:** -- Type definitions: `env`, `term`, `binary`, `resource_type` -- NIF function bindings -- Helper functions: `make_atom`, `make_tuple2`, `get_binary`, `make_binary`, etc. -- Resource management functions - -### 5. Build Configuration (Existing, Updated ✓) -**File:** `native/build.zig` -**Status:** Complete, will link with Lith C ABI - -**Features:** -- Auto-detects Erlang include path -- Links with Lith C ABI (`liblithoglyph.so`) -- Installs to `priv/` directory -- Unit tests included - -## Building - -### Prerequisites - -1. **Zig 0.15.2+** -2. **Erlang/OTP 26+** (with ERTS headers) -3. **Lithoglyph** built with C ABI: - ```bash - cd ~/Documents/hyperpolymath-repos/lith/database/core-forth/ffi/zig - zig build-lib src/abi.zig -dynamic -OReleaseFast - # Creates liblithoglyph.so (or .dylib on macOS, .dll on Windows) - ``` - -### Build Steps - -```bash -cd ~/Documents/hyperpolymath-repos/lithoglyph-beam/native - -# Build NIF (links with Lith C ABI) -zig build -Dlith-path=../../../lith/database/core-forth/ffi/zig/zig-out/lib - -# Output: ../priv/lith_nif.so -``` - -### Run Tests - -```bash -cd native -zig build test -``` - -## Usage Example - -### From Gleam - -```gleam -import lith_beam/client -import gleam/io - -pub fn main() { - // Connect to database - let assert Ok(conn) = client.connect("/tmp/test_lith") - defer client.disconnect(conn) - - // Execute in transaction - let result = client.with_transaction(conn, client.ReadWrite, fn(txn) { - // CBOR-encode operation - let cbor_op = encode_insert_operation() - - // Apply operation - case client.apply_operation(txn, cbor_op) { - Ok(#(result, provenance)) -> { - io.println("Operation succeeded!") - io.debug(provenance) - Ok(result) - } - Error(e) -> { - io.println("Operation failed") - Error(e) - } - } - }) - - case result { - Ok(_) -> io.println("Transaction committed") - Error(e) -> io.debug(e) - } -} - -fn encode_insert_operation() -> BitArray { - // TODO: Use CBOR library to encode operation - // For M10 PoC, manually construct PromptScores CBOR - << - 0xa7, // map(7 pairs) - // "provenance": 95 - 0x6a, 112, 114, 111, 118, 101, 110, 97, 110, 99, 101, - 0x18, 0x5f, - // ... rest of PromptScores fields - >> -} -``` - -### From Erlang - -```erlang --module(lith_example). --export([test/0]). - -test() -> - % Open database - {ok, Db} = lith_nif:db_open(<<"/tmp/test_lith">>), - - % Begin transaction - {ok, Txn} = lith_nif:txn_begin(Db, read_write), - - % Apply operation (CBOR-encoded) - CborOp = encode_insert_op(), - {ok, Result} = lith_nif:apply(Txn, CborOp), - - % Commit - ok = lith_nif:txn_commit(Txn), - - % Close - ok = lith_nif:db_close(Db), - - {ok, Result}. - -encode_insert_op() -> - % CBOR-encode PromptScores map - % TODO: Use cbor library - <<16#a7, ...>>. -``` - -## CBOR Encoding/Decoding (TODO) - -The Gleam client expects CBOR-encoded binaries for operations. We need to add: - -### Option 1: Use Existing CBOR Library -```gleam -import cbor // TODO: Find/create Gleam CBOR library - -pub fn encode_prompt_scores(scores: PromptScores) -> BitArray { - cbor.encode(#( - #("provenance", scores.provenance), - #("replicability", scores.replicability), - // ... - )) -} -``` - -### Option 2: Manual CBOR Construction -See `lith/database/core-forth/test/test-integration.fs` for CBOR byte layout examples. - -## Integration with FormBase - -**FormBase** can now use Lith-BEAM: - -```gleam -// In FormBase backend (Gleam) -import lith_beam/client as lith - -pub fn save_row(conn: lith.Connection, row: Row) -> Result(RowId, Error) { - lith.with_transaction(conn, lith.ReadWrite, fn(txn) { - let cbor_op = encode_row_insert(row) - - case lith.apply_operation(txn, cbor_op) { - Ok(#(result, _provenance)) -> { - let row_id = decode_row_id(result) - Ok(row_id) - } - Error(e) -> Error(LithoglyphError(e)) - } - }) -} -``` - -## Testing Checklist - -### Unit Tests (Zig) -- [ ] NIF lifecycle (init/cleanup) -- [ ] Resource management (DbHandle, TxnHandle) -- [ ] CBOR parsing via Lith C ABI -- [ ] Error handling - -### Integration Tests (Gleam) -- [ ] Connect/disconnect -- [ ] Transaction begin/commit/abort -- [ ] Apply operation with valid CBOR -- [ ] Apply operation with invalid CBOR -- [ ] Schema retrieval -- [ ] Journal retrieval - -### E2E Tests (FormBase) -- [ ] Insert row via Lith-BEAM -- [ ] Query row via Lith-BEAM -- [ ] Update row via Lith-BEAM -- [ ] Delete row via Lith-BEAM - -## Performance Characteristics - -| Operation | Time (M10 PoC) | Notes | -|-----------|---------------|-------| -| `db_open` | ~1ms | Lith init | -| `txn_begin` | < 100μs | Allocate TxnHandle | -| `apply` (parse + persist) | ~10-50ms | Includes gforth subprocess | -| `txn_commit` | < 100μs | Stub (no actual commit yet) | -| `db_close` | ~1ms | Lith cleanup | - -**Future (Embedded gforth):** `apply` < 1ms (eliminate subprocess overhead) - -## Limitations (M10 PoC) - -1. **Subprocess Overhead** - Lith spawns gforth per operation (~10-50ms) -2. **Stub Implementations** - `schema`, `journal`, `txn_commit` return empty/no-op -3. **No CBOR Library** - Manual CBOR encoding required -4. **No Error Details** - Returns atom error codes only -5. **No Connection Pooling** - Each connection = separate Lith instance - -## Next Steps - -### Short Term (1-2 weeks) -1. ✅ Zig NIF implementation - DONE -2. ✅ BEAM API helper - DONE -3. [ ] Build Lith C ABI as shared library -4. [ ] Link NIF with liblithoglyph.so -5. [ ] Test basic operations (connect, apply, disconnect) -6. [ ] Add CBOR encoding/decoding in Gleam -7. [ ] Integration test with FormBase - -### Medium Term (3-4 weeks) -8. [ ] Implement real transaction commit/rollback -9. [ ] Implement schema and journal retrieval -10. [ ] Add connection pooling -11. [ ] Error messages with details (not just atoms) -12. [ ] Performance profiling - -### Long Term -13. [ ] Embed gforth (eliminate subprocess) -14. [ ] Zero-copy CBOR handling -15. [ ] Async operations via BEAM scheduler -16. [ ] Distributed transactions (BEAM cluster) - -## Dependencies - -**Lith-BEAM depends on:** -- Lithoglyph M10 (C ABI) - ✅ Complete -- Lithoglyph M11 (HTTP API) - Not required for direct BEAM usage -- CBOR library for Gleam - TODO (can use manual encoding for now) - -**FormBase depends on:** -- Lith-BEAM - ✅ Ready for integration -- Gleam HTTP server - ✅ Already implemented (80%) -- Grid UI - ✅ Already implemented (95%) - -## Credits - -- **Lith C ABI:** lith/database/core-forth/ffi/zig/src/abi.zig -- **CBOR Specification:** RFC 8949 -- **Erlang NIF:** OTP 26 NIF API -- **Zig:** 0.15.2 -- **Gleam:** 1.x -- **License:** PMPL-1.0-or-later (Palimpsest) -- **Author:** Jonathan D.A. Jewell - ---- - -**Status:** ✅ READY FOR BUILD AND TESTING -**Date:** 2026-02-04 -**Next:** Build liblithoglyph.so and link NIF diff --git a/lithoglyph/beam/M11-HTTP-API-SPEC.md b/lithoglyph/beam/M11-HTTP-API-SPEC.md deleted file mode 100644 index 41ddb71d..00000000 --- a/lithoglyph/beam/M11-HTTP-API-SPEC.md +++ /dev/null @@ -1,451 +0,0 @@ -# M11 HTTP API Specification - -**Date:** 2026-02-04 -**Milestone:** M11 (HTTP API for Lith-Geo & Lith-Analytics) -**Status:** Draft Specification - -## Overview - -M11 adds HTTP/REST API endpoints to Lithoglyph for remote access to: -- Lith-Geo (geospatial data with provenance) -- Lith-Analytics (time-series analytics with provenance) -- Core Lithoglyph operations (transactions, schema, journal) - -## Technology Stack - -**Framework:** Phoenix (Elixir) -- Proven production-ready HTTP server -- Built on BEAM (same runtime as Gleam/Erlang) -- Native support for NIFs -- WebSocket support for real-time journal subscriptions -- Excellent performance (millions of concurrent connections) - -**Alternative (if Phoenix too heavy):** Plug + Cowboy -- Minimal HTTP server -- Direct Erlang/Gleam integration -- Lower overhead - -## API Endpoints - -### Core Lithoglyph Operations - -#### GET /api/v1/version -Get Lithoglyph version - -**Response:** -```json -{ - "version": "1.0.0", - "api_version": "v1" -} -``` - -#### POST /api/v1/databases -Create/open a database - -**Request:** -```json -{ - "path": "/data/mydb", - "mode": "create" | "open" -} -``` - -**Response:** -```json -{ - "database_id": "db_abc123", - "path": "/data/mydb" -} -``` - -#### POST /api/v1/databases/:db_id/transactions -Begin a transaction - -**Request:** -```json -{ - "mode": "read_only" | "read_write" -} -``` - -**Response:** -```json -{ - "transaction_id": "txn_xyz789", - "mode": "read_write" -} -``` - -#### POST /api/v1/transactions/:txn_id/operations -Apply an operation (CBOR-encoded) - -**Request:** -```json -{ - "operation": "", - "provenance": "" -} -``` - -**Response:** -```json -{ - "block_id": "", - "timestamp": "2026-02-04T12:34:56Z" -} -``` - -#### POST /api/v1/transactions/:txn_id/commit -Commit a transaction - -**Response:** -```json -{ - "status": "committed", - "block_count": 42 -} -``` - -#### POST /api/v1/transactions/:txn_id/abort -Abort a transaction - -**Response:** -```json -{ - "status": "aborted" -} -``` - -#### GET /api/v1/databases/:db_id/schema -Get database schema (CBOR) - -**Response:** -```json -{ - "schema": "", - "version": 1 -} -``` - -#### GET /api/v1/databases/:db_id/journal -Get journal entries - -**Query Parameters:** -- `since`: Sequence number (default: 0) -- `limit`: Max entries (default: 100) - -**Response:** -```json -{ - "entries": "", - "next_sequence": 142 -} -``` - -#### DELETE /api/v1/databases/:db_id -Close database - -**Response:** -```json -{ - "status": "closed" -} -``` - -### Lith-Geo Endpoints - -#### POST /api/v1/geo/insert -Insert geospatial data with provenance - -**Request:** -```json -{ - "database_id": "db_abc123", - "geometry": { - "type": "Point", - "coordinates": [-122.4194, 37.7749] - }, - "properties": { - "name": "San Francisco", - "population": 873965 - }, - "provenance": { - "source": "USGS", - "timestamp": "2026-02-04T12:00:00Z", - "confidence": 0.95 - } -} -``` - -**Response:** -```json -{ - "feature_id": "feat_123", - "block_id": "" -} -``` - -#### GET /api/v1/geo/query -Query geospatial data - -**Query Parameters:** -- `bbox`: Bounding box (minx,miny,maxx,maxy) -- `geometry`: GeoJSON geometry filter -- `filter`: Property filter (JSON) -- `limit`: Max results (default: 100) - -**Response:** -```json -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "id": "feat_123", - "geometry": { ... }, - "properties": { ... }, - "provenance": { ... } - } - ] -} -``` - -#### GET /api/v1/geo/features/:feature_id/provenance -Get feature provenance history - -**Response:** -```json -{ - "feature_id": "feat_123", - "provenance_chain": [ - { - "block_id": "", - "timestamp": "2026-02-04T12:00:00Z", - "source": "USGS", - "operation": "insert" - } - ] -} -``` - -### Lith-Analytics Endpoints - -#### POST /api/v1/analytics/timeseries -Insert time-series data with provenance - -**Request:** -```json -{ - "database_id": "db_abc123", - "series_id": "sensor_temp_01", - "timestamp": "2026-02-04T12:34:56Z", - "value": 72.5, - "metadata": { - "sensor_id": "temp_01", - "location": "building_a" - }, - "provenance": { - "source": "iot_gateway", - "quality": "calibrated" - } -} -``` - -**Response:** -```json -{ - "point_id": "ts_456", - "block_id": "" -} -``` - -#### GET /api/v1/analytics/timeseries -Query time-series data - -**Query Parameters:** -- `series_id`: Series identifier -- `start`: Start timestamp (ISO 8601) -- `end`: End timestamp (ISO 8601) -- `aggregation`: none | avg | min | max | sum -- `interval`: Aggregation interval (1m, 5m, 1h, 1d) -- `limit`: Max results (default: 1000) - -**Response:** -```json -{ - "series_id": "sensor_temp_01", - "data": [ - { - "timestamp": "2026-02-04T12:00:00Z", - "value": 72.5, - "provenance": { ... } - } - ] -} -``` - -#### GET /api/v1/analytics/timeseries/:series_id/provenance -Get time-series provenance - -**Response:** -```json -{ - "series_id": "sensor_temp_01", - "provenance_summary": { - "sources": ["iot_gateway", "manual_entry"], - "quality_distribution": { - "calibrated": 0.95, - "uncalibrated": 0.05 - } - } -} -``` - -### Real-Time Subscriptions (WebSocket) - -#### WS /api/v1/journal/subscribe -Subscribe to journal updates - -**Subscribe Message:** -```json -{ - "action": "subscribe", - "database_id": "db_abc123", - "since": 100 -} -``` - -**Update Message (server → client):** -```json -{ - "type": "journal_entry", - "sequence": 101, - "block_id": "", - "timestamp": "2026-02-04T12:34:56Z", - "operation": "" -} -``` - -## Authentication & Authorization - -### M11 PoC: Basic Auth -- HTTP Basic Authentication -- Single admin user for testing - -### M12 Production: JWT + Provenance -- JWT tokens with HMAC-SHA256 -- User provenance tracking -- Permission-based access control -- API key support for service-to-service - -## Error Responses - -Standard error format: -```json -{ - "error": { - "code": "INVALID_OPERATION", - "message": "CBOR parsing failed: invalid major type", - "details": { - "expected": "map", - "received": "array" - } - } -} -``` - -### Error Codes -- `INVALID_OPERATION`: CBOR/operation error -- `TRANSACTION_ERROR`: Transaction state error -- `CONNECTION_ERROR`: Database connection error -- `NOT_FOUND`: Resource not found -- `UNAUTHORIZED`: Authentication required -- `FORBIDDEN`: Permission denied -- `RATE_LIMIT_EXCEEDED`: Too many requests - -## Rate Limiting - -M11 PoC: -- 1000 requests/minute per IP -- 100 concurrent connections per IP - -M12 Production: -- Configurable per-user/per-API-key limits -- Redis-backed distributed rate limiting - -## Implementation Plan - -### Phase 1: Core API (2-3 hours) -1. Phoenix project setup -2. NIF integration (reuse Lith-BEAM) -3. Core endpoints (version, database, transaction, operations) -4. Error handling -5. Basic tests - -### Phase 2: Geo Endpoints (1-2 hours) -1. GeoJSON parsing -2. Spatial query implementation -3. Provenance tracking -4. Bounding box queries - -### Phase 3: Analytics Endpoints (1-2 hours) -1. Time-series data model -2. Aggregation queries -3. Provenance summary -4. Timestamp indexing - -### Phase 4: WebSocket Subscriptions (1 hour) -1. Phoenix Channel setup -2. Journal subscription -3. Real-time updates -4. Connection management - -### Total Estimate: 5-8 hours - -## Testing Strategy - -### Unit Tests -- Each endpoint -- CBOR encoding/decoding -- Error handling - -### Integration Tests -- End-to-end transaction flow -- Geo query with provenance -- Analytics aggregation -- WebSocket subscription - -### Load Tests -- Apache Bench (ab) or wrk -- Target: 10,000 req/s on basic operations -- WebSocket: 10,000 concurrent connections - -## Deployment - -### M11 PoC -- Single Elixir release -- Embedded Erlang VM -- Port 4000 - -### M12 Production -- Distributed Erlang cluster -- Load balancer (nginx/HAProxy) -- HTTPS/TLS -- Health checks -- Metrics (Prometheus) - -## Next Steps - -1. Create Phoenix project: `mix phx.new lith_http --no-html --no-assets --database=false` -2. Add Lith-BEAM NIF dependency -3. Implement core endpoints -4. Add Geo/Analytics modules -5. Write tests -6. Document API with OpenAPI/Swagger - ---- - -**Status:** Specification complete, ready for implementation -**Dependencies:** Lith-BEAM (complete), Phoenix/Plug (to be installed) -**Target:** M11 milestone (HTTP API operational) diff --git a/lithoglyph/beam/MAINTAINERS.adoc b/lithoglyph/beam/MAINTAINERS.adoc deleted file mode 100644 index 48d97817..00000000 --- a/lithoglyph/beam/MAINTAINERS.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Maintainers -:toc: preamble - -This document lists the maintainers of this project and their responsibilities. - -== Current Maintainers - -[cols="2,3,2",options="header"] -|=== -| Name | Role | Contact - -| Jonathan D.A. Jewell -| Lead Maintainer -| https://github.com/hyperpolymath[@hyperpolymath] -|=== - -== Responsibilities - -Maintainers are responsible for: - -* Reviewing and merging pull requests -* Triaging issues and feature requests -* Ensuring code quality and security standards -* Managing releases and versioning -* Upholding the project's code of conduct - -== Becoming a Maintainer - -Contributors who demonstrate: - -* Consistent, high-quality contributions -* Understanding of the project's goals and standards -* Constructive participation in discussions -* Commitment to the project's long-term health - -May be invited to become maintainers at the discretion of existing maintainers. - -== Decision Making - -* Routine decisions (bug fixes, minor improvements) can be made by any maintainer -* Significant changes require discussion and consensus among maintainers -* Breaking changes or major features should be discussed in issues before implementation - -== Contact - -For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/lithoglyph/beam/MOVED.md b/lithoglyph/beam/MOVED.md deleted file mode 100644 index 23194aec..00000000 --- a/lithoglyph/beam/MOVED.md +++ /dev/null @@ -1,12 +0,0 @@ -# This Repository Has Moved - -**This component is now part of the [Lithoglyph monorepo](https://github.com/hyperpolymath/lith).** - -## New Location - -- **Monorepo:** https://github.com/hyperpolymath/lith -- **This Component:** https://github.com/hyperpolymath/lith/tree/main/beamextensions/beambeambeam - -All Lithoglyph development now happens in the monorepo. - -**See you at [github.com/hyperpolymath/lith](https://github.com/hyperpolymath/lith)!** 🚀 diff --git a/lithoglyph/beam/README.adoc b/lithoglyph/beam/README.adoc deleted file mode 100644 index 52fcc78a..00000000 --- a/lithoglyph/beam/README.adoc +++ /dev/null @@ -1,179 +0,0 @@ -image:https://img.shields.io/badge/License-MPL--2.0-blue.svg[License: PMPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] -// SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2025 hyperpolymath - -= Lithoglyph BEAM -:toc: -:icons: font - -BEAM (Erlang/Gleam/Elixir) bindings for https://github.com/hyperpolymath/lith[Lithoglyph] - the narrative-first database. - -== Overview - -Lithoglyph BEAM provides native bindings to Lithoglyph for the BEAM ecosystem: - -* **Gleam** - Type-safe functional interface -* **Erlang** - Low-level NIF bindings -* **Elixir** - (Coming soon) - -All data is passed as CBOR-encoded binaries for maximum efficiency. - -== Installation - -=== Gleam - -Add to your `gleam.toml`: - -[source,toml] ----- -[dependencies] -lith_beam = ">= 0.1.0" ----- - -=== Erlang (rebar3) - -Add to your `rebar.config`: - -[source,erlang] ----- -{deps, [ - {lith_beam, {git, "https://github.com/hyperpolymath/lith-beam.git", {tag, "v0.1.0"}}} -]}. ----- - -== Quick Start - -=== Gleam - -[source,gleam] ----- -import lith_beam/client - -pub fn main() { - // Connect to database - let assert Ok(conn) = client.connect("/path/to/db") - - // Get version - let #(major, minor, patch) = client.version() - io.println("Lithoglyph v" <> int.to_string(major) <> "." <> int.to_string(minor)) - - // Execute in transaction - let assert Ok(result) = client.with_transaction(conn, client.ReadWrite, fn(txn) { - // Your operations here - client.apply_operation(txn, cbor_encoded_op) - }) - - // Close connection - let assert Ok(_) = client.disconnect(conn) -} ----- - -=== Erlang - -[source,erlang] ----- -% Open database -{ok, Db} = lith_nif:db_open(<<"/path/to/db">>), - -% Begin transaction -{ok, Txn} = lith_nif:txn_begin(Db, read_write), - -% Apply operation (CBOR-encoded) -{ok, ResultCbor, ProvenanceCbor} = lith_nif:apply(Txn, OperationCbor), - -% Commit -ok = lith_nif:txn_commit(Txn), - -% Close -ok = lith_nif:db_close(Db). ----- - -== API Reference - -=== Gleam API - -[cols="1,2"] -|=== -| Function | Description - -| `version()` | Get Lithoglyph version as `#(Int, Int, Int)` -| `connect(path)` | Open database at path -| `disconnect(conn)` | Close database connection -| `begin_transaction(conn, mode)` | Start transaction (ReadOnly or ReadWrite) -| `commit(txn)` | Commit transaction -| `abort(txn)` | Abort transaction -| `apply_operation(txn, cbor)` | Apply CBOR-encoded operation -| `get_schema(conn)` | Get database schema (CBOR) -| `get_journal(conn, since)` | Get journal entries since sequence -| `with_transaction(conn, mode, fn)` | Execute with automatic commit/abort -|=== - -=== Erlang NIF API - -[cols="1,2"] -|=== -| Function | Description - -| `version/0` | Get version as `{Major, Minor, Patch}` -| `db_open/1` | Open database (binary path) -| `db_close/1` | Close database -| `txn_begin/2` | Begin transaction with mode atom -| `txn_commit/1` | Commit transaction -| `txn_abort/1` | Abort transaction -| `apply/2` | Apply CBOR operation -| `schema/1` | Get schema -| `journal/2` | Get journal since sequence -|=== - -== Building - -=== Prerequisites - -* Zig >= 0.11.0 -* Erlang/OTP >= 25.0 -* Lithoglyph library - -=== Build NIF - -[source,bash] ----- -cd native -make LITH_PATH=/path/to/lith ----- - -=== Run Tests - -[source,bash] ----- -gleam test ----- - -== Architecture - ----- -┌─────────────────────────────────────┐ -│ Gleam Application │ -│ (lith_beam/client) │ -├─────────────────────────────────────┤ -│ Erlang NIF Module │ -│ (lith_nif.erl) │ -├─────────────────────────────────────┤ -│ Zig NIF Implementation │ -│ (lith_nif.zig) │ -├─────────────────────────────────────┤ -│ Lithoglyph C ABI │ -│ (bridge.zig) │ -├─────────────────────────────────────┤ -│ Lithoglyph Core │ -│ (Forth/Factor) │ -└─────────────────────────────────────┘ ----- - -== License - -PMPL-1.0-or-later - -== Related Projects - -* https://github.com/hyperpolymath/lith[Lithoglyph] - The database engine -* https://github.com/hyperpolymath/formbase[FormBase] - Airtable alternative built on Lithoglyph diff --git a/lithoglyph/beam/ROADMAP.adoc b/lithoglyph/beam/ROADMAP.adoc deleted file mode 100644 index c85340a0..00000000 --- a/lithoglyph/beam/ROADMAP.adoc +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Beam Roadmap - -== Current Status - -Initial development phase. - -== Milestones - -=== v0.1.0 - Foundation -* [ ] Core functionality -* [ ] Basic documentation -* [ ] CI/CD pipeline - -=== v1.0.0 - Stable Release -* [ ] Full feature set -* [ ] Comprehensive tests -* [ ] Production ready - -== Future Directions - -_To be determined based on community feedback._ diff --git a/lithoglyph/beam/RSR_OUTLINE.adoc b/lithoglyph/beam/RSR_OUTLINE.adoc deleted file mode 100644 index ede631fe..00000000 --- a/lithoglyph/beam/RSR_OUTLINE.adoc +++ /dev/null @@ -1,218 +0,0 @@ -= RSR Template Repository - -image:[Palimpsest-MPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] image:[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] -:toc: -:sectnums: - -// Badges -image:https://img.shields.io/badge/RSR-Infrastructure-cd7f32[RSR Infrastructure] -image:https://img.shields.io/badge/Phase-Maintenance-brightgreen[Phase] -image:https://img.shields.io/badge/Guix-Primary-purple?logo=gnu[Guix] - -== Overview - -**The canonical template for RSR (Rhodium Standard Repository) projects.** - -This repository provides the standardized structure, configuration, and tooling for all 139 repos in the hyperpolymath ecosystem. Use it to: - -* Bootstrap new projects with RSR compliance -* Reference the standard directory structure -* Copy configuration templates (Justfile, STATE.scm, etc.) - -== Quick Start - -[source,bash] ----- -# Clone the template -git clone https://github.com/hyperpolymath/RSR-template-repo my-project -cd my-project - -# Remove template git history -rm -rf .git -git init - -# Customize -sed -i 's/RSR-template-repo/my-project/g' Justfile guix.scm README.adoc - -# Enter development environment -guix shell -D -f guix.scm - -# Validate compliance -just validate-rsr ----- - -== What's Included - -[cols="1,3"] -|=== -|File/Directory |Purpose - -|`.editorconfig` -|Editor configuration (indent, charset) - -|`.gitignore` -|Standard ignore patterns - -|`.guix-channel` -|Guix channel definition - -|`.well-known/` -|RFC-compliant metadata (security.txt, ai.txt, humans.txt) - -|`docs/` -|Documentation directory - -|`guix.scm` -|Guix package definition - -|`justfile` -|Task runner with 50+ recipes - -|`LICENSE.txt` -|Palimpsest (PMPL-1.0-or-later) - -|`README.adoc` -|This file - -|`RSR_COMPLIANCE.adoc` -|Compliance tracking - -|`STATE.scm` -|Project state checkpoint -|=== - -== Justfile Features - -The template Justfile provides: - -* **~10 billion recipe combinations** via matrix recipes -* **Cookbook generation**: `just cookbook` → `docs/just-cookbook.adoc` -* **Man page generation**: `just man` → `docs/man/project.1` -* **RSR validation**: `just validate-rsr` -* **STATE.scm management**: `just state-touch`, `just state-phase` -* **Container support**: `just container-build`, `just container-push` -* **CI matrix**: `just ci-matrix [stage] [depth]` - -=== Key Recipes - -[source,bash] ----- -just # Show all recipes -just help # Detailed help -just info # Project info -just combinations # Show matrix options - -just build # Build (debug) -just test # Run tests -just quality # Format + lint + test -just ci # Full CI pipeline - -just validate # RSR + STATE validation -just docs # Generate all docs -just cookbook # Generate Justfile docs - -just guix-shell # Guix dev environment -just container-build # Build container ----- - -== Directory Structure - -[source] ----- -project/ -├── .editorconfig # Editor settings -├── .gitignore # Git ignore -├── .guix-channel # Guix channel -├── .well-known/ # RFC metadata -│ ├── ai.txt -│ ├── humans.txt -│ └── security.txt -├── config/ # Nickel configs (optional) -├── docs/ # Documentation -│ ├── generated/ -│ ├── man/ -│ └── just-cookbook.adoc -├── guix.scm # Guix package -├── Justfile # Task runner -├── LICENSE.txt # Dual license -├── README.adoc # Overview -├── RSR_COMPLIANCE.adoc # Compliance -├── src/ # Source code -├── STATE.scm # State checkpoint -└── tests/ # Tests ----- - -== RSR Compliance - -=== Language Tiers - -* **Tier 1** (Gold): Rust, Elixir, Zig, Ada, Haskell, ReScript -* **Tier 2** (Silver): Nickel, Racket, Guile Scheme, Nix -* **Infrastructure**: Guix channels, derivations - -=== Required Files - -* `.editorconfig` -* `.gitignore` -* `justfile` -* `README.adoc` -* `RSR_COMPLIANCE.adoc` -* `LICENSE` (PMPL-1.0-or-later) -* `.well-known/security.txt` -* `.well-known/ai.txt` -* `.well-known/humans.txt` -* `guix.scm` OR `flake.nix` - -=== Prohibited - -* Python outside `salt/` directory -* TypeScript/JavaScript (use ReScript) -* CUE (use Guile/Nickel) -* `Dockerfile` (use `Containerfile`) - -== STATE.scm - -The STATE.scm file tracks project state: - -[source,scheme] ----- -(define state - `((metadata - (project . "my-project") - (updated . "2025-12-10")) - (position - (phase . implementation) ; design|implementation|testing|maintenance|archived - (maturity . beta)) ; experimental|alpha|beta|production|lts - (ecosystem - (part-of . ("RSR Framework")) - (depends-on . ())))) ----- - -== Badge Schema - -Generate badges from STATE.scm: - -[source,bash] ----- -just badges standard ----- - -See `docs/BADGE_SCHEMA.adoc` for the full badge taxonomy. - -== Ecosystem Integration - -This template is part of: - -* **STATE.scm Ecosystem**: Conversation checkpoints -* **RSR Framework**: Repository standards -* **Consent-Aware-HTTP**: .well-known compliance - -== License - -SPDX-License-Identifier: CC-BY-SA-4.0 - -== Links - -* https://github.com/hyperpolymath/elegant-STATE[elegant-STATE] - STATE.scm tooling -* https://github.com/hyperpolymath/conative-gating[conative-gating] - Policy enforcement -* https://rhodium.sh[Rhodium Standard] - RSR documentation diff --git a/lithoglyph/beam/SECURITY-REQUIREMENTS.scm b/lithoglyph/beam/SECURITY-REQUIREMENTS.scm deleted file mode 100644 index b2b602cb..00000000 --- a/lithoglyph/beam/SECURITY-REQUIREMENTS.scm +++ /dev/null @@ -1,271 +0,0 @@ -;; SPDX-License-Identifier: MPL-2.0 -;; Lith Security Requirements -;; Comprehensive cryptographic and security standards - -(define user-security-requirements - '( - ;; ================================================================= - ;; CRYPTOGRAPHIC PRIMITIVES - ;; ================================================================= - - ;; Password Hashing - (PasswordHashing - (algorithm "Argon2id") - (memory-cost "512 MiB") - (iterations 8) - (parallelism 4) - (rationale "Maximum memory/iterations for GPU/ASIC resistance; aligns with proactive security stance.")) - - ;; General Hashing - (GeneralHashing - (algorithm "SHAKE3-512") - (output-size "512-bit") - (standard "FIPS 202") - (use-cases ("provenance" "key-derivation" "long-term-storage")) - (rationale "Post-quantum secure; use for all hash operations.")) - - ;; Post-Quantum Signatures - (PQSignatures - (primary "Dilithium5-AES") - (standard "ML-DSA-87 (FIPS 204)") - (mode "hybrid") - (classical-component "AES-256") - (fallback "SPHINCS+") - (rationale "Hybrid with AES-256 for belt-and-suspenders security. SPHINCS+ as conservative backup.")) - - ;; Post-Quantum Key Exchange - (PQKeyExchange - (primary "Kyber-1024") - (standard "ML-KEM-1024 (FIPS 203)") - (kdf "SHAKE256-KDF") - (fallback "SPHINCS+") - (rationale "Kyber-1024 for KEM, SHAKE256 for key derivation. SPHINCS+ as backup.")) - - ;; Classical Signatures (Transition) - (ClassicalSignatures - (primary "Ed448") - (hybrid-mode "Ed448 + Dilithium5") - (fallback "SPHINCS+") - (deprecated ("Ed25519" "SHA-1")) - (terminate-immediately ("Ed25519" "SHA-1")) - (rationale "Ed448 for classical compatibility; Dilithium5 for PQ. Terminate Ed25519/SHA-1 immediately.")) - - ;; Symmetric Encryption - (SymmetricEncryption - (algorithm "XChaCha20-Poly1305") - (key-size "256-bit") - (rationale "Larger nonce space; 256-bit keys for quantum margin.")) - - ;; Key Derivation - (KeyDerivation - (algorithm "HKDF-SHAKE512") - (standard "FIPS 202") - (use-cases ("all-secret-key-material")) - (rationale "Post-quantum KDF; use with all secret key material.")) - - ;; Random Number Generation - (RNG - (algorithm "ChaCha20-DRBG") - (seed-size "512-bit") - (standard "SP 800-90Ar1") - (rationale "CSPRNG for deterministic, high-entropy needs.")) - - ;; ================================================================= - ;; USER-FRIENDLY FEATURES - ;; ================================================================= - - (UserFriendlyHashNames - (algorithm "Base32(SHAKE256(hash)) → Wordlist") - (purpose "Memorable, deterministic mapping") - (example "Gigantic-Giraffe-7 for drivers") - (rationale "Human-readable identifiers derived from cryptographic hashes.")) - - ;; ================================================================= - ;; DATABASE & STORAGE - ;; ================================================================= - - (DatabaseHashing - (performance "BLAKE3 (512-bit)") - (long-term-storage "SHAKE3-512") - (semantic-tags ("XML" "ARIA")) - (rationale "BLAKE3 for speed, SHAKE3-512 for long-term storage (semantic XML/ARIA tags).")) - - (SemanticXMLGraphQL - (database "Virtuoso (VOS)") - (query-language "SPARQL 1.2") - (accessibility ("WCAG 2.3 AAA" "ARIA")) - (formal-verification true) - (rationale "Supports WCAG 2.3 AAA, ARIA, and formal verification for accessibility/compliance.")) - - ;; ================================================================= - ;; EXECUTION ENVIRONMENT - ;; ================================================================= - - (VMExecution - (platform "GraalVM") - (formal-verification true) - (rationale "Aligns with preference for introspective, reversible design.")) - - ;; ================================================================= - ;; NETWORK & PROTOCOL - ;; ================================================================= - - (ProtocolStack - (transport "QUIC") - (http "HTTP/3") - (ip "IPv6") - (disabled ("HTTP/1.1" "IPv4" "SHA-1")) - (danger-zone-termination ("HTTP/1.1" "IPv4" "SHA-1")) - (rationale "Terminate HTTP/1.1, IPv4, and SHA-1 per \"danger zone\" policy.")) - - ;; ================================================================= - ;; ACCESSIBILITY & UI - ;; ================================================================= - - (Accessibility - (standard "WCAG 2.3 AAA") - (aria true) - (semantic-xml true) - (design-approach "CSS-first, HTML-second") - (rationale "Full compliance with accessibility requirements.")) - - ;; ================================================================= - ;; CRYPTOGRAPHIC FALLBACK - ;; ================================================================= - - (Fallback - (algorithm "SPHINCS+") - (purpose "Conservative PQ backup for all hybrid classical+PQ systems") - (use-case "If primary PQ algorithm is ever compromised") - (rationale "Belt-and-suspenders approach to post-quantum security.")) - - ;; ================================================================= - ;; FORMAL VERIFICATION - ;; ================================================================= - - (FormalVerification - (tools ("Coq" "Isabelle")) - (purpose "Crypto primitives verification") - (principles ("proactive-attestation" "transparent-logic")) - (rationale "Aligns with system design principles for formal correctness.")) - - ;; ================================================================= - ;; SECURITY POLICY - ;; ================================================================= - - (SecurityPolicy - (stance "Proactive") - (quantum-resistance "Required") - (hybrid-approach "Classical + PQ for all critical operations") - (termination-policy "Immediate removal of deprecated algorithms") - (belt-and-suspenders "Multiple layers of security (primary + fallback)")) - )) - -;; ================================================================= -;; IMPLEMENTATION PRIORITIES FOR LITHOGLYPH -;; ================================================================= - -(define lith-security-implementation - '( - ;; M11: Current Milestone - (M11 - (focus "HTTP API security") - (requirements - (https "TLS 1.3 with QUIC") - (authentication "JWT with HMAC-SHA256 (transition to SHAKE512)") - (rate-limiting "Redis-backed distributed rate limiting") - (cbor-validation "Strict CBOR major type checking"))) - - ;; M12: Production Security - (M12 - (focus "Post-quantum cryptography integration") - (requirements - (signatures "Ed448 + Dilithium5 hybrid") - (key-exchange "Kyber-1024 with SHAKE256-KDF") - (symmetric "XChaCha20-Poly1305") - (password-hashing "Argon2id (512 MiB, 8 iter, 4 lanes)") - (provenance-hashing "SHAKE3-512"))) - - ;; M13: Full PQ Migration - (M13 - (focus "Complete post-quantum migration") - (requirements - (terminate ("Ed25519" "SHA-1" "SHA-256")) - (upgrade-all-systems "Dilithium5 + SPHINCS+ fallback") - (formal-verification "Coq proofs for all crypto primitives"))) - - ;; M14: Advanced Security - (M14 - (focus "GraalVM integration + formal verification") - (requirements - (vm "GraalVM with formal verification") - (accessibility "WCAG 2.3 AAA + ARIA") - (semantic-web "Virtuoso VOS + SPARQL 1.2") - (network "QUIC + HTTP/3 + IPv6 only"))) - )) - -;; ================================================================= -;; LITHOGLYPH CRYPTO LIBRARY STACK -;; ================================================================= - -(define lith-crypto-stack - '( - (layer "Password Hashing") - (library "rust-argon2") - (config "512 MiB memory, 8 iterations, 4 lanes") - - (layer "General Hashing") - (library "tiny-keccak (SHAKE3)") - (config "512-bit output") - - (layer "Post-Quantum Signatures") - (library "pqcrypto-dilithium (ML-DSA)") - (variant "Dilithium5-AES") - (fallback "pqcrypto-sphincsplus (SPHINCS+)") - - (layer "Post-Quantum KEM") - (library "pqcrypto-kyber (ML-KEM)") - (variant "Kyber-1024") - (kdf "tiny-keccak (SHAKE256)") - - (layer "Symmetric Encryption") - (library "chacha20poly1305") - (variant "XChaCha20-Poly1305") - (key-size "256-bit") - - (layer "Classical Signatures") - (library "ed25519-dalek") - (variant "Ed448") - (note "Transition to hybrid Ed448 + Dilithium5") - - (layer "Key Derivation") - (library "hkdf + tiny-keccak") - (algorithm "HKDF-SHAKE512") - - (layer "Random Number Generation") - (library "rand_chacha") - (algorithm "ChaCha20-DRBG") - (seed-size "512-bit") - )) - -;; ================================================================= -;; DANGER ZONE - IMMEDIATE TERMINATION -;; ================================================================= - -(define algorithms-to-terminate-immediately - '( - "SHA-1" ;; Broken - "Ed25519" ;; Inadequate for PQ era - "MD5" ;; Completely broken - "HTTP/1.1" ;; Replaced by HTTP/3 - "IPv4" ;; Replaced by IPv6 - )) - -;; ================================================================= -;; EXPORT -;; ================================================================= - -(provide 'user-security-requirements - 'lith-security-implementation - 'lith-crypto-stack - 'algorithms-to-terminate-immediately) diff --git a/lithoglyph/beam/SESSION-SUMMARY-M10-DAY3.md b/lithoglyph/beam/SESSION-SUMMARY-M10-DAY3.md deleted file mode 100644 index b82a612c..00000000 --- a/lithoglyph/beam/SESSION-SUMMARY-M10-DAY3.md +++ /dev/null @@ -1,265 +0,0 @@ -# M10 Day 3 Session Summary - -**Date:** 2026-02-04 -**Duration:** ~8 hours -**Status:** ALL PRIORITIES COMPLETE ✅ - -## Accomplishments - -### ✅ Priority 1: Lith-BEAM Rustler Migration (2 hours) -**Goal:** Replace non-working Zig NIF with production-ready Rust/Rustler implementation - -**Results:** -- Migrated from Zig to Rustler 0.35 -- Implemented all 9 NIF functions: - - `version()`, `db_open()`, `db_close()` - - `txn_begin()`, `txn_commit()`, `txn_abort()` - - `apply()`, `schema()`, `journal()` -- Fixed return type issues (removed nested ok tuples) -- All tests passing ✓ -- M10 PoC stub implementations working - -**Technology:** Rust + Rustler 0.35 -**Lines of Code:** ~165 (lib.rs) -**Test Results:** 8/8 passing - -### ✅ Priority 2: FormBase Integration Testing (3 hours) -**Goal:** Test Lith-BEAM NIF with actual Gleam FormBase client - -**Results:** -- Copied working Rustler NIF to FormBase -- Created Gleam FFI wrapper (`lith/nif_ffi.gleam`) -- Updated Gleam client (`lith/client.gleam`) to use real NIF -- Fixed type system issues (dynamic.Dynamic handling) -- Added error cases (ParseFailed, InvalidHandle) -- Compiled Gleam project successfully -- All NIF tests passing from FormBase ✓ - -**Technology:** Gleam + Erlang FFI + Rustler -**Lines of Code:** ~600 (Gleam client + FFI + Erlang wrapper) -**Test Results:** 8/8 passing - -### ✅ Priority 3: M11 HTTP API Specification (2 hours) -**Goal:** Design HTTP/REST API for Lith-Geo and Lith-Analytics - -**Results:** -- Complete API specification written -- Core endpoints defined (version, database, transaction, operations) -- Geo endpoints specified (insert, query, provenance) -- Analytics endpoints specified (timeseries insert, query, aggregation) -- WebSocket subscription design (real-time journal updates) -- Authentication/authorization plan (Basic Auth → JWT) -- Error handling specification -- Rate limiting design -- Implementation plan (5-8 hours) - -**Technology:** Phoenix/Elixir (recommended) -**Documentation:** M11-HTTP-API-SPEC.md (1900+ lines) - -### ✅ Bonus: Security Requirements Integration -**Goal:** Capture comprehensive security requirements for Lithoglyph ecosystem - -**Results:** -- Post-quantum cryptography roadmap -- Algorithm specifications (Dilithium5, Kyber-1024, SHAKE3-512) -- Password hashing (Argon2id, 512 MiB, 8 iter) -- Symmetric encryption (XChaCha20-Poly1305) -- Network protocol (QUIC + HTTP/3 + IPv6) -- Accessibility (WCAG 2.3 AAA + ARIA) -- Formal verification (Coq/Isabelle) -- Danger zone termination list (SHA-1, Ed25519, HTTP/1.1, IPv4) - -**Documentation:** SECURITY-REQUIREMENTS.scm - -## Technical Highlights - -### Zig → Rustler Migration -**Why it was needed:** -- Zig NIF had persistent segfault during `erlang:load_nif/2` -- Deep ABI incompatibility between Zig struct layout and Erlang expectations -- Inline functions required complex C shim layer - -**Why Rustler won:** -- Proven BEAM compatibility (used in production Elixir apps) -- Clean resource management -- No ABI mismatch issues -- Excellent error handling -- Migration took exactly as estimated (2 hours) - -### CBOR Validation -Implemented in both Zig and Rust versions: -```rust -// Check first byte is CBOR map (major type 5) -let first_byte = op_cbor[0]; -let major_type = (first_byte >> 5) & 0x07; -if major_type != 5 { - return Err(atoms::parse_failed()); -} -``` - -### Gleam FFI Integration -Used `@external` declarations to call Erlang NIFs: -```gleam -@external(erlang, "lith_nif", "version") -pub fn nif_version() -> #(Int, Int, Int) -``` - -Handled Erlang result tuples (`{ok, Value}` or `{error, Reason}`): -```gleam -fn handle_erlang_result(result: dynamic.Dynamic) - -> Result(dynamic.Dynamic, dynamic.Dynamic) -``` - -## Files Created/Modified - -### Lith-BEAM Repository -- `native_rust/src/lib.rs` (created) - Rustler NIF implementation -- `native_rust/Cargo.toml` (created) - Rust dependencies -- `src/lith_nif.erl` (created) - Erlang NIF wrapper -- `test_rust.erl` (created) - Rust NIF test script -- `BUILD-STATUS.md` (updated) - Build status documentation -- `M11-HTTP-API-SPEC.md` (created) - M11 API specification -- `SECURITY-REQUIREMENTS.scm` (created) - Security requirements -- `SESSION-SUMMARY-M10-DAY3.md` (this file) - -### FormBase Repository -- `server/native_rust/` (copied) - Rustler NIF -- `server/src/lith_nif.erl` (updated) - Erlang NIF wrapper -- `server/src/lith/nif_ffi.gleam` (created) - Gleam FFI declarations -- `server/src/lith/client.gleam` (updated) - Real NIF integration -- `server/src/router.gleam` (updated) - Added error cases -- `server/src/formbase_server.gleam` (updated) - Added error cases -- `server/test_lith_nif.erl` (created) - Erlang test script -- `server/LITH-INTEGRATION.md` (updated) - Integration status - -## Test Results - -### Lith-BEAM (Rustler) -``` -Test 1: Version {1,0,0} ✓ -Test 2: Database opened ✓ -Test 3: Transaction started ✓ -Test 4: Operation applied, block ID: [0,0,0,0,0,0,0,1] ✓ -Test 5: Transaction committed ✓ -Test 6: Schema: CBOR empty map ✓ -Test 7: Journal: CBOR empty array ✓ -Test 8: Database closed ✓ -=== All tests passed! === -``` - -### FormBase (Gleam Integration) -``` -Test 1: Version {1,0,0} ✓ -Test 2: Database opened ✓ -Test 3: Transaction started ✓ -Test 4: Operation applied, block ID: [0,0,0,0,0,0,0,1] ✓ -Test 5: Transaction committed ✓ -Test 6: Schema: CBOR empty map ✓ -Test 7: Journal: CBOR empty array ✓ -Test 8: Database closed ✓ -=== All tests passed! === -``` - -## Performance (M10 PoC Stubs) - -| Operation | Time | Notes | -|-----------|------|-------| -| `version()` | < 1μs | Direct return | -| `db_open()` | < 10μs | Creates resource | -| `txn_begin()` | < 5μs | Creates resource | -| `apply()` | < 50μs | CBOR validation only | -| `txn_commit()` | < 5μs | Returns atom | - -## Next Steps - -### M11 Implementation (5-8 hours) -1. Create Phoenix project -2. Integrate Lith-BEAM NIF -3. Implement core HTTP endpoints -4. Add Geo endpoints -5. Add Analytics endpoints -6. WebSocket subscriptions -7. Write integration tests -8. Deploy PoC - -### M12 Production Security (8-12 hours) -1. Implement post-quantum cryptography -2. Migrate to Ed448 + Dilithium5 hybrid -3. Add Kyber-1024 key exchange -4. Replace SHA-256 with SHAKE3-512 -5. Implement Argon2id password hashing -6. Formal verification with Coq -7. Accessibility compliance (WCAG 2.3 AAA) - -### M13 Full Integration (Future) -1. Replace M10 stubs with real Lithoglyph C ABI calls -2. Add gforth subprocess integration -3. Performance optimization -4. Distributed deployment -5. Production monitoring - -## Lessons Learned - -### 1. Trust Proven Technologies -Rustler saved ~4 hours of debugging compared to Zig. When dealing with FFI/ABI, use battle-tested libraries. - -### 2. API Design First -Spending 2 hours on M11 API specification will save 10+ hours of refactoring later. - -### 3. Type Systems Matter -Gleam's strict type system caught many errors at compile-time that would have been runtime bugs. - -### 4. Test Early, Test Often -Creating tests alongside implementation (not after) caught issues immediately. - -### 5. Security Upfront -Defining security requirements now (M10) prevents costly retrofits in M12+. - -## Repository Status - -### Lith Ecosystem (8 repos) -| Repo | Status | Ready for M11 | -|------|--------|---------------| -| **lithoglyph** (core-forth) | Complete | ✅ (C ABI built) | -| **lithoglyph-beam** | Complete | ✅ (Rustler NIF working) | -| **formbase** | Complete | ✅ (Gleam client integrated) | -| lithoglyph-geo | Spec only | ⏳ (needs M11 HTTP API) | -| lithoglyph-analytics | Spec only | ⏳ (needs M11 HTTP API) | -| lithoglyph-debugger | Not started | ⏳ | -| lithoglyph-studio | Not started | ⏳ | -| gql-dt | Complete | ✅ (GQL parser) | - -## Time Investment - -| Priority | Estimated | Actual | Difference | -|----------|-----------|--------|------------| -| Priority 1 (Rustler) | 2-3 hours | 2 hours | ✓ On target | -| Priority 2 (FormBase) | 1-2 hours | 3 hours | +1 hour (Gleam FFI complexity) | -| Priority 3 (M11 Spec) | 2-3 hours | 2 hours | ✓ On target | -| **Total** | **5-8 hours** | **7 hours** | ✓ Within estimate | - -## Conclusion - -M10 Day 3 was highly productive: -- ✅ Fixed Lith-BEAM NIF (Zig → Rustler migration) -- ✅ Integrated with FormBase (Gleam client working) -- ✅ Specified M11 HTTP API (complete design) -- ✅ Documented security requirements (PQ crypto roadmap) - -**All three priorities completed within estimated time.** - -Lithoglyph ecosystem is now ready for M11 HTTP API implementation, which will enable: -- Remote access to Lithoglyph -- Geospatial data with provenance (Lith-Geo) -- Time-series analytics with provenance (Lith-Analytics) -- Real-time journal subscriptions (WebSocket) - -**Next session:** Implement M11 HTTP API using Phoenix/Elixir. - ---- - -**Session End:** 2026-02-04 -**Total Commits:** 15+ -**Lines of Code:** ~2400 -**Tests Passing:** 16/16 ✓ -**Coffee Consumed:** ☕☕☕☕ diff --git a/lithoglyph/beam/docs/CITATIONS.adoc b/lithoglyph/beam/docs/CITATIONS.adoc deleted file mode 100644 index 6f167bdf..00000000 --- a/lithoglyph/beam/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {PMPL-1.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/lithoglyph/beam/examples/SafeDOMExample.affine b/lithoglyph/beam/examples/SafeDOMExample.affine deleted file mode 100644 index 2a62c1d1..00000000 --- a/lithoglyph/beam/examples/SafeDOMExample.affine +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SafeDOMExample.affine — formally-verified DOM mounting (aspirational). -// -// This example shows the *shape* of SafeDOM consumer code in current -// AffineScript syntax. The `SafeDOM` stdlib surface it references -// (`mount_safe`, `mount_when_ready`, `mount_batch`, -// `proven_selector_validate`, `proven_html_validate`, `mount`) is the -// target of `affinescript#56` (DOM+Pixi binding survey) and does not -// yet exist in the published stdlib. The file is therefore -// parse-checked but not type-checked end-to-end until #56 lands the -// bindings; `affinescript check` reports `Resolve.UndefinedModule -// SafeDOM` which is expected. -// -// Previous versions of this file (estate-wide, 5 dialect variants) -// pre-dated ADR-014 (qualified paths), ADR-016 (effect rows), and the -// `#{`-record-literal sigil (ADR-215). They were retired in favour of -// this canonical via the gitbot-fleet#208 sweep (2026-05-26). - -module SafeDOMExample; - -use prelude::{Option, Some, None, Result, Ok, Err}; - -// `Element` and friends are nominal extern types for now — the real -// shape lands with affinescript#56. -extern type Element; -extern type Selector; -extern type ValidHTML; - -// Single-mount status, lifted from the host into a typed tag union. -enum MountStatus { - Mounted(Element), - MountPointNotFound(String), - InvalidSelector(String), - InvalidHTML(String) -} - -// Batch-mount result. -enum MountResult { - Mounted([Element]), - Failed(String) -} - -// Spec for one element in a batch mount. -struct MountSpec { - selector: String, - html: String -} - -// SafeDOM's host-side surface, all IO-effecting. Callbacks are passed -// as separate parameters (rather than a `MountCallbacks` record) -// because fn-typed struct fields are not currently parser-supported. -extern fn mount_safe( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_when_ready( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_batch(specs: ref [MountSpec]) -{IO}-> MountResult; - -extern fn proven_selector_validate(s: ref String) -{IO}-> Result; -extern fn proven_html_validate(s: ref String) -{IO}-> Result; -extern fn mount(sel: ref Selector, html: ref ValidHTML) -{IO}-> MountStatus; - -extern fn array_for_each(xs: ref [Element], f: fn(Element) -> ()) -{IO}-> (); -extern fn array_len(xs: ref [Element]) -> Int; - -// Example 1 — basic mount with success/error branches. -pub fn mount_app() -{IO}-> () { - mount_safe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - fn(el) -> () { Console::log("App mounted successfully"); }, - fn(err) -> () { Console::error("Mount failed: " ++ err); }, - ); -} - -// Example 2 — defer until DOM ready. -pub fn mount_when_dom_ready() -{IO}-> () { - mount_when_ready( - "#app", - "

App Title

", - fn(_el) -> () { Console::log("Mounted after DOM ready"); }, - fn(err) -> () { Console::error("Failed: " ++ err); }, - ); -} - -// Example 3 — atomic batch mount. -pub fn mount_multiple() -{IO}-> () { - let specs = [ - MountSpec #{ selector: "#header", html: "

Site Title

" }, - MountSpec #{ selector: "#nav", html: "" }, - MountSpec #{ selector: "#main", html: "

Content here

" }, - MountSpec #{ selector: "#footer", html: "
2026
" }, - ]; - - match mount_batch(specs) { - Mounted(elements) => { - Console::log("Batch mount succeeded"); - array_for_each(elements, fn(_el) -> () { Console::log(" element"); }); - }, - Failed(err) => { - Console::error("Batch mount failed (atomic — none mounted): " ++ err); - } - } -} - -// Example 4 — explicit two-stage validation before mounting. -pub fn mount_with_validation() -{IO}-> () { - match proven_selector_validate("#my-app") { - Err(e) => Console::error("Invalid selector: " ++ e), - Ok(valid_selector) => match proven_html_validate("
Content
") { - Err(e) => Console::error("Invalid HTML: " ++ e), - Ok(valid_html) => match mount(valid_selector, valid_html) { - Mounted(_el) => Console::log("Mounted with validated inputs"), - MountPointNotFound(s) => Console::error("Element not found: " ++ s), - InvalidSelector(_) => Console::error("impossible — already validated"), - InvalidHTML(_) => Console::error("impossible — already validated"), - }, - }, - } -} diff --git a/lithoglyph/beam/examples/web-project-deno.json b/lithoglyph/beam/examples/web-project-deno.json deleted file mode 100644 index 5ddd3bd7..00000000 --- a/lithoglyph/beam/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "^12.0.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/lithoglyph/beam/ffi/zig/build.zig b/lithoglyph/beam/ffi/zig/build.zig deleted file mode 100644 index 4a2e049a..00000000 --- a/lithoglyph/beam/ffi/zig/build.zig +++ /dev/null @@ -1,94 +0,0 @@ -// {{PROJECT}} FFI Build Configuration -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Shared library (.so, .dylib, .dll) - const lib = b.addSharedLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Set version - lib.version = .{ .major = 0, .minor = 1, .patch = 0 }; - - // Static library (.a) - const lib_static = b.addStaticLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Install artifacts - b.installArtifact(lib); - b.installArtifact(lib_static); - - // Generate header file for C compatibility - const header = b.addInstallHeader( - b.path("include/{{project}}.h"), - "{{project}}.h", - ); - b.getInstallStep().dependOn(&header.step); - - // Unit tests - const lib_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const run_lib_tests = b.addRunArtifact(lib_tests); - - const test_step = b.step("test", "Run library tests"); - test_step.dependOn(&run_lib_tests.step); - - // Integration tests - const integration_tests = b.addTest(.{ - .root_source_file = b.path("test/integration_test.zig"), - .target = target, - .optimize = optimize, - }); - - integration_tests.linkLibrary(lib); - - const run_integration_tests = b.addRunArtifact(integration_tests); - - const integration_test_step = b.step("test-integration", "Run integration tests"); - integration_test_step.dependOn(&run_integration_tests.step); - - // Documentation - const docs = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = .Debug, - }); - - const docs_step = b.step("docs", "Generate documentation"); - docs_step.dependOn(&b.addInstallDirectory(.{ - .source_dir = docs.getEmittedDocs(), - .install_dir = .prefix, - .install_subdir = "docs", - }).step); - - // Benchmark (if needed) - const bench = b.addExecutable(.{ - .name = "{{project}}-bench", - .root_source_file = b.path("bench/bench.zig"), - .target = target, - .optimize = .ReleaseFast, - }); - - bench.linkLibrary(lib); - - const run_bench = b.addRunArtifact(bench); - - const bench_step = b.step("bench", "Run benchmarks"); - bench_step.dependOn(&run_bench.step); -} diff --git a/lithoglyph/beam/ffi/zig/src/main.zig b/lithoglyph/beam/ffi/zig/src/main.zig deleted file mode 100644 index 6b233bc7..00000000 --- a/lithoglyph/beam/ffi/zig/src/main.zig +++ /dev/null @@ -1,274 +0,0 @@ -// {{PROJECT}} FFI Implementation -// -// This module implements the C-compatible FFI declared in src/abi/Foreign.idr -// All types and layouts must match the Idris2 ABI definitions. -// -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -// Version information (keep in sync with project) -const VERSION = "0.1.0"; -const BUILD_INFO = "{{PROJECT}} built with Zig " ++ @import("builtin").zig_version_string; - -/// Thread-local error storage -threadlocal var last_error: ?[]const u8 = null; - -/// Set the last error message -fn setError(msg: []const u8) void { - last_error = msg; -} - -/// Clear the last error -fn clearError() void { - last_error = null; -} - -//============================================================================== -// Core Types (must match src/abi/Types.idr) -//============================================================================== - -/// Result codes (must match Idris2 Result type) -pub const Result = enum(c_int) { - ok = 0, - @"error" = 1, - invalid_param = 2, - out_of_memory = 3, - null_pointer = 4, -}; - -/// Library handle (opaque to prevent direct access) -pub const Handle = opaque { - // Internal state hidden from C - allocator: std.mem.Allocator, - initialized: bool, - // Add your fields here -}; - -//============================================================================== -// Library Lifecycle -//============================================================================== - -/// Initialize the library -/// Returns a handle, or null on failure -export fn {{project}}_init() ?*Handle { - const allocator = std.heap.c_allocator; - - const handle = allocator.create(Handle) catch { - setError("Failed to allocate handle"); - return null; - }; - - // Initialize handle - handle.* = .{ - .allocator = allocator, - .initialized = true, - }; - - clearError(); - return handle; -} - -/// Free the library handle -export fn {{project}}_free(handle: ?*Handle) void { - const h = handle orelse return; - const allocator = h.allocator; - - // Clean up resources - h.initialized = false; - - allocator.destroy(h); - clearError(); -} - -//============================================================================== -// Core Operations -//============================================================================== - -/// Process data (example operation) -export fn {{project}}_process(handle: ?*Handle, input: u32) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Example processing logic - _ = input; - - clearError(); - return .ok; -} - -//============================================================================== -// String Operations -//============================================================================== - -/// Get a string result (example) -/// Caller must free the returned string -export fn {{project}}_get_string(handle: ?*Handle) ?[*:0]const u8 { - const h = handle orelse { - setError("Null handle"); - return null; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return null; - } - - // Example: allocate and return a string - const result = h.allocator.dupeZ(u8, "Example result") catch { - setError("Failed to allocate string"); - return null; - }; - - clearError(); - return result.ptr; -} - -/// Free a string allocated by the library -export fn {{project}}_free_string(str: ?[*:0]const u8) void { - const s = str orelse return; - const allocator = std.heap.c_allocator; - - const slice = std.mem.span(s); - allocator.free(slice); -} - -//============================================================================== -// Array/Buffer Operations -//============================================================================== - -/// Process an array of data -export fn {{project}}_process_array( - handle: ?*Handle, - buffer: ?[*]const u8, - len: u32, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const buf = buffer orelse { - setError("Null buffer"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Access the buffer - const data = buf[0..len]; - _ = data; - - // Process data here - - clearError(); - return .ok; -} - -//============================================================================== -// Error Handling -//============================================================================== - -/// Get the last error message -/// Returns null if no error -export fn {{project}}_last_error() ?[*:0]const u8 { - const err = last_error orelse return null; - - // Return C string (static storage, no need to free) - const allocator = std.heap.c_allocator; - const c_str = allocator.dupeZ(u8, err) catch return null; - return c_str.ptr; -} - -//============================================================================== -// Version Information -//============================================================================== - -/// Get the library version -export fn {{project}}_version() [*:0]const u8 { - return VERSION.ptr; -} - -/// Get build information -export fn {{project}}_build_info() [*:0]const u8 { - return BUILD_INFO.ptr; -} - -//============================================================================== -// Callback Support -//============================================================================== - -/// Callback function type (C ABI) -pub const Callback = *const fn (u64, u32) callconv(.C) u32; - -/// Register a callback -export fn {{project}}_register_callback( - handle: ?*Handle, - callback: ?Callback, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const cb = callback orelse { - setError("Null callback"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Store callback for later use - _ = cb; - - clearError(); - return .ok; -} - -//============================================================================== -// Utility Functions -//============================================================================== - -/// Check if handle is initialized -export fn {{project}}_is_initialized(handle: ?*Handle) u32 { - const h = handle orelse return 0; - return if (h.initialized) 1 else 0; -} - -//============================================================================== -// Tests -//============================================================================== - -test "lifecycle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try std.testing.expect({{project}}_is_initialized(handle) == 1); -} - -test "error handling" { - const result = {{project}}_process(null, 0); - try std.testing.expectEqual(Result.null_pointer, result); - - const err = {{project}}_last_error(); - try std.testing.expect(err != null); -} - -test "version" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - try std.testing.expectEqualStrings(VERSION, ver_str); -} diff --git a/lithoglyph/beam/ffi/zig/test/integration_test.zig b/lithoglyph/beam/ffi/zig/test/integration_test.zig deleted file mode 100644 index 03419949..00000000 --- a/lithoglyph/beam/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,182 +0,0 @@ -// {{PROJECT}} Integration Tests -// SPDX-License-Identifier: MPL-2.0 -// -// These tests verify that the Zig FFI correctly implements the Idris2 ABI - -const std = @import("std"); -const testing = std.testing; - -// Import FFI functions -extern fn {{project}}_init() ?*opaque {}; -extern fn {{project}}_free(?*opaque {}) void; -extern fn {{project}}_process(?*opaque {}, u32) c_int; -extern fn {{project}}_get_string(?*opaque {}) ?[*:0]const u8; -extern fn {{project}}_free_string(?[*:0]const u8) void; -extern fn {{project}}_last_error() ?[*:0]const u8; -extern fn {{project}}_version() [*:0]const u8; -extern fn {{project}}_is_initialized(?*opaque {}) u32; - -//============================================================================== -// Lifecycle Tests -//============================================================================== - -test "create and destroy handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try testing.expect(handle != null); -} - -test "handle is initialized" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const initialized = {{project}}_is_initialized(handle); - try testing.expectEqual(@as(u32, 1), initialized); -} - -test "null handle is not initialized" { - const initialized = {{project}}_is_initialized(null); - try testing.expectEqual(@as(u32, 0), initialized); -} - -//============================================================================== -// Operation Tests -//============================================================================== - -test "process with valid handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const result = {{project}}_process(handle, 42); - try testing.expectEqual(@as(c_int, 0), result); // 0 = ok -} - -test "process with null handle returns error" { - const result = {{project}}_process(null, 42); - try testing.expectEqual(@as(c_int, 4), result); // 4 = null_pointer -} - -//============================================================================== -// String Tests -//============================================================================== - -test "get string result" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const str = {{project}}_get_string(handle); - defer if (str) |s| {{project}}_free_string(s); - - try testing.expect(str != null); -} - -test "get string with null handle" { - const str = {{project}}_get_string(null); - try testing.expect(str == null); -} - -//============================================================================== -// Error Handling Tests -//============================================================================== - -test "last error after null handle operation" { - _ = {{project}}_process(null, 0); - - const err = {{project}}_last_error(); - try testing.expect(err != null); - - if (err) |e| { - const err_str = std.mem.span(e); - try testing.expect(err_str.len > 0); - } -} - -test "no error after successful operation" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - _ = {{project}}_process(handle, 0); - - // Error should be cleared after successful operation - // (This depends on implementation) -} - -//============================================================================== -// Version Tests -//============================================================================== - -test "version string is not empty" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - try testing.expect(ver_str.len > 0); -} - -test "version string is semantic version format" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - // Should be in format X.Y.Z - try testing.expect(std.mem.count(u8, ver_str, ".") >= 1); -} - -//============================================================================== -// Memory Safety Tests -//============================================================================== - -test "multiple handles are independent" { - const h1 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h1); - - const h2 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h2); - - try testing.expect(h1 != h2); - - // Operations on h1 should not affect h2 - _ = {{project}}_process(h1, 1); - _ = {{project}}_process(h2, 2); -} - -test "double free is safe" { - const handle = {{project}}_init() orelse return error.InitFailed; - - {{project}}_free(handle); - {{project}}_free(handle); // Should not crash -} - -test "free null is safe" { - {{project}}_free(null); // Should not crash -} - -//============================================================================== -// Thread Safety Tests (if applicable) -//============================================================================== - -test "concurrent operations" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const ThreadContext = struct { - h: *opaque {}, - id: u32, - }; - - const thread_fn = struct { - fn run(ctx: ThreadContext) void { - _ = {{project}}_process(ctx.h, ctx.id); - } - }.run; - - var threads: [4]std.Thread = undefined; - for (&threads, 0..) |*thread, i| { - thread.* = try std.Thread.spawn(.{}, thread_fn, .{ - ThreadContext{ .h = handle, .id = @intCast(i) }, - }); - } - - for (threads) |thread| { - thread.join(); - } -} diff --git a/lithoglyph/beam/gleam.toml b/lithoglyph/beam/gleam.toml deleted file mode 100644 index 521a4ea0..00000000 --- a/lithoglyph/beam/gleam.toml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 - -name = "lith_beam" -version = "0.1.0" -description = "BEAM bindings for Lithoglyph - the narrative-first database" -# PMPL-1.0-or-later preferred; MPL-2.0 required for valid SPDX in Gleam ecosystem -licences = ["MPL-2.0"] -repository = { type = "github", user = "hyperpolymath", repo = "lith-beam" } - -[dependencies] -gleam_stdlib = ">= 0.34.0 and < 2.0.0" -gleam_erlang = ">= 0.25.0 and < 1.0.0" - -[dev-dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/lithoglyph/beam/native/Justfile b/lithoglyph/beam/native/Justfile deleted file mode 100644 index cecaaa7c..00000000 --- a/lithoglyph/beam/native/Justfile +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Justfile for Lith NIF - -set shell := ["bash", "-euo", "pipefail", "-c"] - -priv_dir := "../priv" - -default: - @just --list --unsorted - -all: - @mkdir -p {{priv_dir}} - @ERL_INCLUDE_PATH="$$(erl -noshell -eval 'io:format("~s/usr/include", [code:root_dir()])' -s init stop)" \ - zig build -Doptimize=ReleaseSafe - -lith: - @lith_path="${LITH_PATH:-$HOME/Documents/hyperpolymath-repos/lith}" - cd "$$lith_path/core-zig" && zig build - -erl: - erlc -o ../ebin src/lith_nif.erl - -clean: - rm -rf zig-cache zig-out {{priv_dir}}/lith_nif.so - -test: - zig build test - -install: all - @erl_lib_path="$$(erl -noshell -eval 'io:format("~s/usr/lib", [code:root_dir()])' -s init stop)" - cp {{priv_dir}}/lith_nif.so "$$erl_lib_path/" - -dev: all erl - @echo "NIF built successfully" - @echo "Library: {{priv_dir}}/lith_nif.so" diff --git a/lithoglyph/beam/native/build.zig b/lithoglyph/beam/native/build.zig deleted file mode 100644 index 0ae04ffa..00000000 --- a/lithoglyph/beam/native/build.zig +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Build configuration for Lith NIF - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Find Erlang include directory - const erl_include = blk: { - // Try to get from environment - if (std.process.getEnvVarOwned(b.allocator, "ERL_INCLUDE_PATH")) |path| { - break :blk path; - } else |_| {} - - // Try to find via erl command - const result = std.process.Child.run(.{ - .allocator = b.allocator, - .argv = &.{ "erl", "-noshell", "-eval", "io:format(\"~s\", [code:root_dir()])", "-s", "init", "stop" }, - }) catch { - @panic("Failed to find Erlang installation. Set ERL_INCLUDE_PATH."); - }; - - const root_dir = std.mem.trim(u8, result.stdout, &std.ascii.whitespace); - break :blk b.fmt("{s}/usr/include", .{root_dir}); - }; - - // Build the NIF shared library - const root_module = b.createModule(.{ - .root_source_file = .{ .cwd_relative = "src/lith_nif.zig" }, - .target = target, - .optimize = optimize, - .link_libc = true, - }); - - const lib = b.addLibrary(.{ - .name = "lith_nif", - .root_module = root_module, - .linkage = .dynamic, - }); - - // Add Erlang NIF headers - lib.root_module.addIncludePath(.{ .cwd_relative = erl_include }); - - // Add C helper file for NIF inline functions - lib.addCSourceFile(.{ - .file = .{ .cwd_relative = "src/nif_helpers.c" }, - .flags = &.{"-std=c11"}, - }); - - // Link against Lith - // In production, this would link against liblith.so - // For now, we'll add a stub or expect Lith to be linked separately - if (b.option([]const u8, "lith-path", "Path to Lith library")) |lith_path| { - lib.root_module.addLibraryPath(.{ .cwd_relative = lith_path }); - lib.root_module.linkSystemLibrary("lith", .{}); - } - - // Install to priv directory - const install = b.addInstallArtifact(lib, .{ - .dest_dir = .{ .override = .{ .custom = "../priv" } }, - }); - - b.getInstallStep().dependOn(&install.step); - - // Tests - const test_module = b.createModule(.{ - .root_source_file = .{ .cwd_relative = "src/lith_nif.zig" }, - .target = target, - .optimize = optimize, - }); - - const unit_tests = b.addTest(.{ - .root_module = test_module, - }); - - unit_tests.root_module.addIncludePath(.{ .cwd_relative = erl_include }); - - const run_unit_tests = b.addRunArtifact(unit_tests); - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_unit_tests.step); -} diff --git a/lithoglyph/beam/native/src/beam.zig b/lithoglyph/beam/native/src/beam.zig deleted file mode 100644 index cec8af3d..00000000 --- a/lithoglyph/beam/native/src/beam.zig +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// BEAM NIF API - Zig bindings for Erlang NIF -// -// Provides Zig-friendly wrappers for the Erlang NIF C API - -const std = @import("std"); - -// Opaque types from erl_nif.h (avoid @cImport to prevent inline function issues) -pub const env = opaque {}; -pub const term = c_ulong; - -// Binary structure (matches ErlNifBinary) -pub const binary = extern struct { - size: usize, - data: [*]u8, -}; - -// Resource type (opaque) -pub const resource_type = opaque {}; - -// NIF function pointer -pub const ErlNifFunc = extern struct { - name: [*:0]const u8, - arity: c_uint, - fptr: *const fn (?*env, c_int, [*c]const term) callconv(.c) term, - flags: c_uint, -}; - -// Resource type initialization -pub const ErlNifResourceTypeInit = extern struct { - dtor: ?*const fn (?*env, ?*anyopaque) callconv(.c) void, - stop: ?*const fn (?*env, ?*anyopaque, term, [*c]c_int) callconv(.c) void, - down: ?*const fn (?*env, ?*anyopaque, ?*anyopaque, ?*anyopaque) callconv(.c) void, -}; - -// NIF entry structure -pub const ErlNifEntry = extern struct { - major: c_int, - minor: c_int, - name: [*:0]const u8, - num_of_funcs: c_int, - funcs: [*c]const ErlNifFunc, - load: ?*const fn (?*env, [*c]?*anyopaque, term) callconv(.c) c_int, - reload: ?*const fn (?*env, [*c]?*anyopaque, term) callconv(.c) c_int, - upgrade: ?*const fn (?*env, [*c]?*anyopaque, [*c]?*anyopaque, term) callconv(.c) c_int, - unload: ?*const fn (?*env, ?*anyopaque) callconv(.c) void, - vm_variant: [*:0]const u8, - options: c_uint, - sizeof_ErlNifResourceTypeInit: usize, -}; - -// NIF API version -pub const ERL_NIF_MAJOR_VERSION = 2; -pub const ERL_NIF_MINOR_VERSION = 16; - -// Encoding constants -const ERL_NIF_LATIN1 = 1; - -// Resource flags -const ERL_NIF_RT_CREATE = 1; -const ERL_NIF_RT_TAKEOVER = 2; - -// External NIF functions (direct C ABI) -extern fn enif_make_atom(env: ?*env, name: [*:0]const u8) callconv(.c) term; -extern fn enif_make_badarg(env: ?*env) callconv(.c) term; -extern fn enif_make_int(env: ?*env, i: c_int) callconv(.c) term; -extern fn enif_get_atom(env: ?*env, t: term, buf: [*]u8, len: c_uint, encoding: c_uint) callconv(.c) c_uint; -extern fn enif_get_resource(env: ?*env, t: term, resource_type: ?*resource_type, objp: *?*anyopaque) callconv(.c) c_int; -extern fn enif_alloc_resource(resource_type: ?*resource_type, size: usize) callconv(.c) ?*anyopaque; -extern fn enif_release_resource(obj: *anyopaque) callconv(.c) void; -extern fn enif_make_resource(env: ?*env, obj: *anyopaque) callconv(.c) term; -// Destructor function type for resources -pub const ErlNifResourceDtor = fn (?*env, ?*anyopaque) callconv(.c) void; - -extern fn enif_open_resource_type(env: ?*env, module_str: ?[*:0]const u8, name: [*:0]const u8, dtor: ?*const ErlNifResourceDtor, flags: c_uint, tried: ?*c_uint) callconv(.c) ?*resource_type; -extern fn enif_inspect_binary(env: ?*env, term: term, bin: *binary) callconv(.c) c_int; -extern fn enif_alloc_binary(size: usize, bin: *binary) callconv(.c) c_int; -extern fn enif_make_binary(env: ?*env, bin: *const binary) callconv(.c) term; - -// C helper functions (from nif_helpers.c - wrappers for inline functions) -extern fn nif_make_tuple2(env: ?*env, t1: term, t2: term) callconv(.c) term; -extern fn nif_make_tuple3(env: ?*env, t1: term, t2: term, t3: term) callconv(.c) term; - -// Zig-friendly wrappers -pub fn make_atom(e: ?*env, name: [*:0]const u8) term { - return enif_make_atom(e, name); -} - -pub fn make_badarg(e: ?*env) term { - return enif_make_badarg(e); -} - -pub fn make_int(e: ?*env, i: c_int) term { - return enif_make_int(e, i); -} - -pub fn make_tuple2(e: ?*env, t1: term, t2: term) term { - return nif_make_tuple2(e, t1, t2); -} - -pub fn make_tuple3(e: ?*env, t1: term, t2: term, t3: term) term { - return nif_make_tuple3(e, t1, t2, t3); -} - -pub fn get_atom(e: ?*env, t: term, buf: []u8) usize { - const len = enif_get_atom(e, t, buf.ptr, @intCast(buf.len), ERL_NIF_LATIN1); - return if (len > 0) @intCast(len - 1) else 0; -} - -pub fn get_binary(e: ?*env, t: term, bin: *binary) c_int { - return enif_inspect_binary(e, t, bin); -} - -pub fn make_binary(e: ?*env, data: []const u8) !term { - var bin: binary = undefined; - if (enif_alloc_binary(data.len, &bin) == 0) { - return error.AllocFailed; - } - @memcpy(bin.data[0..data.len], data); - return enif_make_binary(e, &bin); -} - -pub fn get_resource(e: ?*env, t: term, comptime T: type, rt: ?*resource_type) !*anyopaque { - _ = T; - var obj: ?*anyopaque = null; - if (enif_get_resource(e, t, rt, &obj) == 0) { - return error.InvalidResource; - } - return obj orelse error.NullResource; -} - -pub fn alloc_resource(e: ?*env, obj: anytype, rt: ?*resource_type) !term { - const T = @TypeOf(obj.*); - const res_ptr = enif_alloc_resource(rt, @sizeOf(T)) orelse return error.AllocFailed; - // SAFETY: res_ptr comes from enif_alloc_resource() which allocates @sizeOf(T) - // bytes with alignment sufficient for any C type. Since T is a Zig struct used - // as a NIF resource, the BEAM allocator guarantees at least max_align_t alignment - // which satisfies @alignOf(T). The pointer is valid until enif_release_resource(). - const typed_ptr: *T = @ptrCast(@alignCast(res_ptr)); - typed_ptr.* = obj.*; - return enif_make_resource(e, res_ptr); -} - -pub fn open_resource_type(e: ?*env, name: [*:0]const u8, dtor: ?*const ErlNifResourceDtor) !?*resource_type { - const rt = enif_open_resource_type(e, null, name, dtor, ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, null); - return rt orelse error.OpenFailed; -} diff --git a/lithoglyph/beam/native/src/lith_nif.erl b/lithoglyph/beam/native/src/lith_nif.erl deleted file mode 100644 index 5ff26df6..00000000 --- a/lithoglyph/beam/native/src/lith_nif.erl +++ /dev/null @@ -1,107 +0,0 @@ -%% SPDX-License-Identifier: MPL-2.0 -%% Lith NIF - Erlang interface to Lith -%% -%% This module loads the Zig NIF and provides Erlang functions -%% to interact with Lith. - --module(lith_nif). --export([ - version/0, - db_open/1, - db_close/1, - txn_begin/2, - txn_commit/1, - txn_abort/1, - apply/2, - schema/1, - journal/2 -]). - --on_load(init/0). - --define(NIF_NOT_LOADED, erlang:nif_error(nif_not_loaded)). - -%% @doc Initialize the NIF -init() -> - PrivDir = case code:priv_dir(formbase_server) of - {error, _} -> - %% Fallback for development - case code:which(?MODULE) of - Filename when is_list(Filename) -> - filename:join([filename:dirname(Filename), "..", "priv"]); - _ -> - "priv" - end; - Dir -> - Dir - end, - SoPath = filename:join(PrivDir, "lith_nif"), - erlang:load_nif(SoPath, 0). - -%% @doc Get Lith version as {Major, Minor, Patch} --spec version() -> {non_neg_integer(), non_neg_integer(), non_neg_integer()}. -version() -> - ?NIF_NOT_LOADED. - -%% @doc Open a Lith database -%% @param Path Binary path to the database directory -%% @returns {ok, DbRef} | {error, Reason} --spec db_open(binary()) -> {ok, reference()} | {error, atom()}. -db_open(_Path) -> - ?NIF_NOT_LOADED. - -%% @doc Close a Lith database -%% @param DbRef Database reference from db_open/1 -%% @returns ok | {error, Reason} --spec db_close(reference()) -> ok | {error, atom()}. -db_close(_DbRef) -> - ?NIF_NOT_LOADED. - -%% @doc Begin a transaction -%% @param DbRef Database reference -%% @param Mode Transaction mode: read_only | read_write -%% @returns {ok, TxnRef} | {error, Reason} --spec txn_begin(reference(), read_only | read_write) -> {ok, reference()} | {error, atom()}. -txn_begin(_DbRef, _Mode) -> - ?NIF_NOT_LOADED. - -%% @doc Commit a transaction -%% @param TxnRef Transaction reference -%% @returns ok | {error, Reason} --spec txn_commit(reference()) -> ok | {error, atom()}. -txn_commit(_TxnRef) -> - ?NIF_NOT_LOADED. - -%% @doc Abort a transaction -%% @param TxnRef Transaction reference -%% @returns ok --spec txn_abort(reference()) -> ok. -txn_abort(_TxnRef) -> - ?NIF_NOT_LOADED. - -%% @doc Apply an operation within a transaction -%% @param TxnRef Transaction reference -%% @param OpCbor CBOR-encoded operation -%% @returns {ok, ResultCbor} | {ok, ResultCbor, ProvenanceCbor} | {error, Reason} --spec apply(reference(), binary()) -> - {ok, binary()} | - {ok, binary(), binary()} | - {error, atom()} | - {error, atom(), binary()}. -apply(_TxnRef, _OpCbor) -> - ?NIF_NOT_LOADED. - -%% @doc Get database schema -%% @param DbRef Database reference -%% @returns {ok, SchemaCbor} | {error, Reason} --spec schema(reference()) -> {ok, binary()} | {error, atom()}. -schema(_DbRef) -> - ?NIF_NOT_LOADED. - -%% @doc Get journal entries since a sequence number -%% @param DbRef Database reference -%% @param Since Sequence number to start from -%% @returns {ok, JournalCbor} | {error, Reason} --spec journal(reference(), non_neg_integer()) -> {ok, binary()} | {error, atom()}. -journal(_DbRef, _Since) -> - ?NIF_NOT_LOADED. diff --git a/lithoglyph/beam/native/src/lith_nif.zig b/lithoglyph/beam/native/src/lith_nif.zig deleted file mode 100644 index e3fa75a2..00000000 --- a/lithoglyph/beam/native/src/lith_nif.zig +++ /dev/null @@ -1,823 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Lith BEAM NIF - Zig implementation -// -// This NIF connects BEAM (Erlang/Gleam/Elixir) to Lith via the Lith C ABI. -// Uses CBOR-encoded binaries for efficient data transfer. -// -// The NIF delegates all storage operations to the core bridge (generated/abi/bridge.h) -// via C ABI extern declarations. Error codes from the bridge are translated to -// BEAM-style {:ok, result} / {:error, reason} tuples. - -const std = @import("std"); -const beam = @import("beam.zig"); - -// ============================================================ -// Lith C ABI declarations (from generated/abi/bridge.h) -// -// These extern functions are provided by liblith.so (the Zig FFI -// bridge implementation at ffi/zig/src/bridge.zig). The NIF shared -// library links against liblith at load time. -// ============================================================ - -/// Opaque database handle from the core bridge -const LithDb = opaque {}; - -/// Opaque transaction handle from the core bridge -const LithTxn = opaque {}; - -/// Owned byte buffer passed across the FFI boundary -const LgBlob = extern struct { - ptr: ?[*]const u8, - len: usize, -}; - -/// Result type for operations returning data + provenance -const LgResult = extern struct { - data: LgBlob, - provenance: LgBlob, - status: c_int, // LithStatus - error_blob: LgBlob, -}; - -/// Transaction mode -const LgTxnMode = enum(c_int) { - read_only = 0, - read_write = 1, -}; - -/// Render options for introspection functions -const LgRenderOpts = extern struct { - format: c_int, // 0 = JSON - include_metadata: bool, -}; - -/// LithStatus codes (must match bridge.h LithStatus enum) -const LithStatus = enum(c_int) { - ok = 0, - err_internal = 1, - err_not_found = 2, - err_invalid_argument = 3, - err_out_of_memory = 4, - err_not_implemented = 5, - err_txn_not_active = 6, - err_txn_already_committed = 7, - err_io_error = 8, - err_corruption = 9, - err_conflict = 10, - err_already_exists = 11, -}; - -// --- Extern C ABI bridge functions --- - -extern fn lith_db_open( - path_ptr: [*]const u8, - path_len: usize, - opts_ptr: ?[*]const u8, - opts_len: usize, - out_db: *?*LithDb, - out_err: *LgBlob, -) callconv(.c) c_int; - -extern fn lith_db_close(db: *LithDb) callconv(.c) c_int; - -extern fn lith_txn_begin( - db: *LithDb, - mode: LgTxnMode, - out_txn: *?*LithTxn, - out_err: *LgBlob, -) callconv(.c) c_int; - -extern fn lith_txn_commit(txn: *LithTxn, out_err: *LgBlob) callconv(.c) c_int; - -extern fn lith_txn_abort(txn: *LithTxn) callconv(.c) c_int; - -extern fn lith_apply( - txn: *LithTxn, - op_ptr: [*]const u8, - op_len: usize, -) callconv(.c) LgResult; - -extern fn lith_introspect_schema( - db: *LithDb, - out_schema: *LgBlob, - out_err: *LgBlob, -) callconv(.c) c_int; - -extern fn lith_render_journal( - db: *LithDb, - since: u64, - opts: LgRenderOpts, - out_text: *LgBlob, - out_err: *LgBlob, -) callconv(.c) c_int; - -extern fn lith_blob_free(blob: *LgBlob) callconv(.c) void; - -extern fn lith_version() callconv(.c) u32; - -/// Convert an LithStatus integer to an atom name for BEAM error tuples. -/// Returns a descriptive atom string for each known status code. -fn status_to_atom(status: c_int) [*:0]const u8 { - return switch (status) { - 0 => "ok", - 1 => "internal_error", - 2 => "not_found", - 3 => "invalid_argument", - 4 => "out_of_memory", - 5 => "not_implemented", - 6 => "txn_not_active", - 7 => "txn_already_committed", - 8 => "io_error", - 9 => "corruption", - 10 => "conflict", - 11 => "already_exists", - else => "unknown_error", - }; -} - -// Global allocator -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -const allocator = gpa.allocator(); - -// Resource types (initialized in nif_init) -var db_handle_type: ?*beam.resource_type = undefined; -var txn_handle_type: ?*beam.resource_type = undefined; - -// Database handle wrapper — holds an opaque LithDb pointer from the core bridge. -const DbHandle = struct { - lith: *LithDb, - path: []const u8, - - /// Open a database via the core bridge C ABI (lith_db_open). - fn create(path: []const u8) !*DbHandle { - var out_db: ?*LithDb = null; - var out_err: LgBlob = .{ .ptr = null, .len = 0 }; - - const status = lith_db_open( - path.ptr, - path.len, - null, // no options - 0, - &out_db, - &out_err, - ); - - // Free error blob regardless of outcome - if (out_err.ptr != null) lith_blob_free(&out_err); - - if (status != @intFromEnum(LithStatus.ok) or out_db == null) { - return error.InitFailed; - } - - const db = try allocator.create(DbHandle); - db.* = .{ - .lith = out_db.?, - .path = try allocator.dupe(u8, path), - }; - - return db; - } - - /// Close the database via the core bridge C ABI (lith_db_close) and free memory. - fn destroy(self: *DbHandle) void { - _ = lith_db_close(self.lith); - allocator.free(self.path); - allocator.destroy(self); - } -}; - -// Transaction handle wrapper — holds an opaque LithTxn pointer from the core bridge. -const TxnHandle = struct { - lith_txn: *LithTxn, - db: *DbHandle, - mode: TransactionMode, - - const TransactionMode = enum { - read_only, - read_write, - - /// Convert to the C ABI LgTxnMode enum for bridge calls. - fn to_lg_mode(self: TransactionMode) LgTxnMode { - return switch (self) { - .read_only => .read_only, - .read_write => .read_write, - }; - } - }; -}; - -//============================================================================== -// NIF Functions -//============================================================================== - -/// Get Lith version from the core bridge. -/// The bridge encodes version as: major * 10000 + minor * 100 + patch. -/// Returns {Major, Minor, Patch} tuple. -export fn version(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - _ = argc; - _ = argv; - - const ver = lith_version(); - const major: c_int = @intCast(ver / 10000); - const minor: c_int = @intCast((ver % 10000) / 100); - const patch: c_int = @intCast(ver % 100); - - return beam.make_tuple3(env, - beam.make_int(env, major), - beam.make_int(env, minor), - beam.make_int(env, patch), - ); -} - -/// Open a Lith database -/// Parameters: Path (binary) -/// Returns: {ok, DbRef} | {error, Reason} -export fn db_open(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 1) { - return beam.make_badarg(env); - } - - // Get path as binary - var path_bin: beam.binary = undefined; - if (beam.get_binary(env, argv[0], &path_bin) == 0) { - return beam.make_badarg(env); - } - - const path = path_bin.data[0..path_bin.size]; - - // Create database handle - const db = DbHandle.create(path) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "init_failed") - ); - }; - - // Create resource (opaque reference for Erlang) - const db_res = beam.alloc_resource(env, db, db_handle_type) catch { - db.destroy(); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "resource_alloc_failed") - ); - }; - - return beam.make_tuple2(env, - beam.make_atom(env, "ok"), - db_res - ); -} - -/// Close a Lith database -/// Parameters: DbRef (resource) -/// Returns: ok | {error, Reason} -export fn db_close(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 1) { - return beam.make_badarg(env); - } - - // Get database handle - const db_ptr = beam.get_resource(env, argv[0], DbHandle, db_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle") - ); - }; - - // SAFETY: db_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in db_open. The NIF resource system - // guarantees the pointer is valid while the resource reference is live. Alignment - // is met because DbHandle was heap-allocated by allocator.create(DbHandle). - const db: *DbHandle = @ptrCast(@alignCast(db_ptr)); - db.destroy(); - - return beam.make_atom(env, "ok"); -} - -/// Begin a transaction -/// Parameters: DbRef, Mode (read_only | read_write) -/// Returns: {ok, TxnRef} | {error, Reason} -export fn txn_begin(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 2) { - return beam.make_badarg(env); - } - - // Get database handle - const db_ptr = beam.get_resource(env, argv[0], DbHandle, db_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle") - ); - }; - - // SAFETY: db_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in db_open. The NIF resource system - // guarantees the pointer is valid while the resource reference is live. Alignment - // is met because DbHandle was heap-allocated by allocator.create(DbHandle). - const db: *DbHandle = @ptrCast(@alignCast(db_ptr)); - - // Get transaction mode - var mode_atom: [32]u8 = undefined; - const mode_len = beam.get_atom(env, argv[1], &mode_atom); - if (mode_len == 0) { - return beam.make_badarg(env); - } - - const mode_str = mode_atom[0..mode_len]; - const mode: TxnHandle.TransactionMode = if (std.mem.eql(u8, mode_str, "read_only")) - .read_only - else if (std.mem.eql(u8, mode_str, "read_write")) - .read_write - else - return beam.make_badarg(env); - - // Begin transaction via the core bridge C ABI (lith_txn_begin) - var out_txn: ?*LithTxn = null; - var out_err: LgBlob = .{ .ptr = null, .len = 0 }; - - const status = lith_txn_begin( - db.lith, - mode.to_lg_mode(), - &out_txn, - &out_err, - ); - - // Free error blob regardless of outcome - if (out_err.ptr != null) lith_blob_free(&out_err); - - if (status != @intFromEnum(LithStatus.ok) or out_txn == null) { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(status)), - ); - } - - // Create transaction handle wrapping the bridge LithTxn - const txn = allocator.create(TxnHandle) catch { - // Abort the bridge transaction to avoid leaking it - _ = lith_txn_abort(out_txn.?); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "alloc_failed"), - ); - }; - - txn.* = .{ - .lith_txn = out_txn.?, - .db = db, - .mode = mode, - }; - - // Create resource - const txn_res = beam.alloc_resource(env, txn, txn_handle_type) catch { - _ = lith_txn_abort(out_txn.?); - allocator.destroy(txn); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "resource_alloc_failed"), - ); - }; - - return beam.make_tuple2(env, - beam.make_atom(env, "ok"), - txn_res, - ); -} - -/// Commit a transaction -/// Parameters: TxnRef -/// Returns: ok | {error, Reason} -export fn txn_commit(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 1) { - return beam.make_badarg(env); - } - - // Get transaction handle - const txn_ptr = beam.get_resource(env, argv[0], TxnHandle, txn_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle") - ); - }; - - // SAFETY: txn_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in txn_begin. The NIF resource - // system guarantees the pointer is valid while the resource reference is live. - // Alignment is met because TxnHandle was heap-allocated by allocator.create(). - const txn: *TxnHandle = @ptrCast(@alignCast(txn_ptr)); - - // Commit the transaction via the core bridge C ABI (lith_txn_commit). - // This executes the 6-phase WAL: journal -> sync -> blocks -> deletes -> superblock -> sync. - var out_err: LgBlob = .{ .ptr = null, .len = 0 }; - const status = lith_txn_commit(txn.lith_txn, &out_err); - - // Free error blob regardless of outcome - if (out_err.ptr != null) lith_blob_free(&out_err); - - // Clean up the Zig-side wrapper regardless of commit outcome - allocator.destroy(txn); - - if (status != @intFromEnum(LithStatus.ok)) { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(status)), - ); - } - - return beam.make_atom(env, "ok"); -} - -/// Abort a transaction, discarding all buffered operations. -/// Parameters: TxnRef -/// Returns: ok | {error, Reason} -export fn txn_abort(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 1) { - return beam.make_badarg(env); - } - - // Get transaction handle - const txn_ptr = beam.get_resource(env, argv[0], TxnHandle, txn_handle_type) catch { - return beam.make_atom(env, "ok"); // Already aborted/invalid - }; - - // SAFETY: txn_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in txn_begin. The NIF resource - // system guarantees the pointer is valid while the resource reference is live. - // Alignment is met because TxnHandle was heap-allocated by allocator.create(). - const txn: *TxnHandle = @ptrCast(@alignCast(txn_ptr)); - - // Abort the transaction via the core bridge C ABI (lith_txn_abort). - const status = lith_txn_abort(txn.lith_txn); - - // Clean up the Zig-side wrapper regardless of abort outcome - allocator.destroy(txn); - - if (status != @intFromEnum(LithStatus.ok)) { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(status)), - ); - } - - return beam.make_atom(env, "ok"); -} - -/// Apply an operation within a transaction via the core bridge. -/// The bridge buffers writes until commit (6-phase WAL). -/// Parameters: TxnRef, OpCbor (binary) -/// Returns: {ok, ResultCbor} | {ok, ResultCbor, ProvenanceCbor} | {error, Reason} -export fn apply(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 2) { - return beam.make_badarg(env); - } - - // Get transaction handle - const txn_ptr = beam.get_resource(env, argv[0], TxnHandle, txn_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle"), - ); - }; - - // SAFETY: txn_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in txn_begin. The NIF resource - // system guarantees the pointer is valid while the resource reference is live. - // Alignment is met because TxnHandle was heap-allocated by allocator.create(). - const txn: *TxnHandle = @ptrCast(@alignCast(txn_ptr)); - - // Get CBOR operation binary - var cbor_bin: beam.binary = undefined; - if (beam.get_binary(env, argv[1], &cbor_bin) == 0) { - return beam.make_badarg(env); - } - - if (cbor_bin.size == 0 or cbor_bin.size > 1_048_576) { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_argument"), - ); - } - - const cbor_data = cbor_bin.data[0..cbor_bin.size]; - - // Apply the operation via the core bridge C ABI (lith_apply). - // The bridge parses the CBOR/JSON payload, validates it, and buffers the write. - const result: LgResult = lith_apply(txn.lith_txn, cbor_data.ptr, cbor_data.len); - - if (result.status != @intFromEnum(LithStatus.ok)) { - // Free any blobs the bridge may have allocated - var err_blob = result.error_blob; - if (err_blob.ptr != null) lith_blob_free(&err_blob); - var data_blob = result.data; - if (data_blob.ptr != null) lith_blob_free(&data_blob); - - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(result.status)), - ); - } - - // Build the result binary from the bridge data blob - const result_bin = blk: { - if (result.data.ptr != null and result.data.len > 0) { - // SAFETY: result.data.ptr is non-null and result.data.len > 0 (checked above). - // The bridge guarantees the pointer is valid until lith_blob_free() is called. - // We copy the data into a BEAM binary before freeing, so no use-after-free. - const data_slice = result.data.ptr.?[0..result.data.len]; - break :blk beam.make_binary(env, data_slice) catch { - var data_blob = result.data; - lith_blob_free(&data_blob); - var prov_blob = result.provenance; - if (prov_blob.ptr != null) lith_blob_free(&prov_blob); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "result_alloc_failed"), - ); - }; - } else { - // No data returned — return empty binary - const empty = [_]u8{}; - break :blk beam.make_binary(env, &empty) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "result_alloc_failed"), - ); - }; - } - }; - - // Include provenance token in response if the bridge provided one - const has_provenance = result.provenance.ptr != null and result.provenance.len > 0; - - if (has_provenance) { - // SAFETY: result.provenance.ptr is non-null and result.provenance.len > 0 (checked above). - // The bridge guarantees the pointer is valid until lith_blob_free() is called. - // We copy the data into a BEAM binary before freeing. - const prov_slice = result.provenance.ptr.?[0..result.provenance.len]; - const prov_bin = beam.make_binary(env, prov_slice) catch { - var data_blob = result.data; - if (data_blob.ptr != null) lith_blob_free(&data_blob); - var prov_blob = result.provenance; - lith_blob_free(&prov_blob); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "result_alloc_failed"), - ); - }; - - // Free bridge blobs now that data is copied into BEAM binaries - var data_blob = result.data; - if (data_blob.ptr != null) lith_blob_free(&data_blob); - var prov_blob = result.provenance; - lith_blob_free(&prov_blob); - - return beam.make_tuple3(env, - beam.make_atom(env, "ok"), - result_bin, - prov_bin, - ); - } - - // Free bridge data blob (no provenance to free) - var data_blob = result.data; - if (data_blob.ptr != null) lith_blob_free(&data_blob); - - return beam.make_tuple2(env, - beam.make_atom(env, "ok"), - result_bin, - ); -} - -/// Get database schema via the core bridge (lith_introspect_schema). -/// Parameters: DbRef -/// Returns: {ok, SchemaJson} | {error, Reason} -export fn schema(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 1) { - return beam.make_badarg(env); - } - - // Get database handle from resource - const db_ptr = beam.get_resource(env, argv[0], DbHandle, db_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle"), - ); - }; - - // SAFETY: db_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in db_open. The NIF resource system - // guarantees the pointer is valid while the resource reference is live. Alignment - // is met because DbHandle was heap-allocated by allocator.create(DbHandle). - const db: *DbHandle = @ptrCast(@alignCast(db_ptr)); - - // Call the core bridge to introspect the schema - var out_schema: LgBlob = .{ .ptr = null, .len = 0 }; - var out_err: LgBlob = .{ .ptr = null, .len = 0 }; - - const status = lith_introspect_schema(db.lith, &out_schema, &out_err); - - // Free error blob regardless of outcome - if (out_err.ptr != null) lith_blob_free(&out_err); - - if (status != @intFromEnum(LithStatus.ok)) { - if (out_schema.ptr != null) lith_blob_free(&out_schema); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(status)), - ); - } - - // Copy schema data into a BEAM binary - const schema_bin = blk: { - if (out_schema.ptr != null and out_schema.len > 0) { - // SAFETY: out_schema.ptr is non-null and out_schema.len > 0 (checked above). - // The bridge guarantees the pointer is valid until lith_blob_free() is called. - const schema_slice = out_schema.ptr.?[0..out_schema.len]; - break :blk beam.make_binary(env, schema_slice) catch { - lith_blob_free(&out_schema); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "alloc_failed"), - ); - }; - } else { - // No schema data — return empty JSON object - const empty_map = [_]u8{ '{', '}' }; - break :blk beam.make_binary(env, &empty_map) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "alloc_failed"), - ); - }; - } - }; - - // Free the bridge blob now that data is copied - if (out_schema.ptr != null) lith_blob_free(&out_schema); - - return beam.make_tuple2(env, - beam.make_atom(env, "ok"), - schema_bin, - ); -} - -/// Get journal entries since a sequence number via the core bridge (lith_render_journal). -/// Parameters: DbRef, Since (integer) -/// Returns: {ok, JournalJson} | {error, Reason} -export fn journal(env: ?*beam.env, argc: c_int, argv: [*c]const beam.term) beam.term { - if (argc != 2) { - return beam.make_badarg(env); - } - - // Get database handle from resource - const db_ptr = beam.get_resource(env, argv[0], DbHandle, db_handle_type) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "invalid_handle"), - ); - }; - - // SAFETY: db_ptr comes from beam.get_resource() which retrieves the pointer - // originally stored by beam.alloc_resource() in db_open. The NIF resource system - // guarantees the pointer is valid while the resource reference is live. Alignment - // is met because DbHandle was heap-allocated by allocator.create(DbHandle). - const db: *DbHandle = @ptrCast(@alignCast(db_ptr)); - - // Get the 'since' sequence number - var since_val: c_int = undefined; - // Use beam.make_int to check — we need to extract the integer from term. - // The NIF C API provides enif_get_int for this. - since_val = nif_get_int(env, argv[1]) orelse { - return beam.make_badarg(env); - }; - - const since: u64 = if (since_val >= 0) - @intCast(since_val) - else - 0; // Negative since is treated as "from the beginning" - - // Render journal entries via the core bridge - const opts: LgRenderOpts = .{ - .format = 0, // JSON - .include_metadata = true, - }; - var out_text: LgBlob = .{ .ptr = null, .len = 0 }; - var out_err: LgBlob = .{ .ptr = null, .len = 0 }; - - const status = lith_render_journal(db.lith, since, opts, &out_text, &out_err); - - // Free error blob regardless of outcome - if (out_err.ptr != null) lith_blob_free(&out_err); - - if (status != @intFromEnum(LithStatus.ok)) { - if (out_text.ptr != null) lith_blob_free(&out_text); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, status_to_atom(status)), - ); - } - - // Copy journal data into a BEAM binary - const journal_bin = blk: { - if (out_text.ptr != null and out_text.len > 0) { - // SAFETY: out_text.ptr is non-null and out_text.len > 0 (checked above). - // The bridge guarantees the pointer is valid until lith_blob_free() is called. - const text_slice = out_text.ptr.?[0..out_text.len]; - break :blk beam.make_binary(env, text_slice) catch { - lith_blob_free(&out_text); - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "alloc_failed"), - ); - }; - } else { - // No journal entries — return empty JSON array - const empty_array = [_]u8{ '[', ']' }; - break :blk beam.make_binary(env, &empty_array) catch { - return beam.make_tuple2(env, - beam.make_atom(env, "error"), - beam.make_atom(env, "alloc_failed"), - ); - }; - } - }; - - // Free the bridge blob now that data is copied - if (out_text.ptr != null) lith_blob_free(&out_text); - - return beam.make_tuple2(env, - beam.make_atom(env, "ok"), - journal_bin, - ); -} - -/// Helper: extract a C int from a BEAM term. Returns null if the term is not an integer. -fn nif_get_int(e: ?*beam.env, t: beam.term) ?c_int { - var val: c_int = undefined; - if (enif_get_int(e, t, &val) == 0) { - return null; - } - return val; -} - -// Additional extern needed for integer extraction -extern fn enif_get_int(env: ?*beam.env, term: beam.term, ip: *c_int) callconv(.c) c_int; - -//============================================================================== -// NIF Initialization -//============================================================================== - -const nif_funcs = [_]beam.ErlNifFunc{ - .{ .name = "version", .arity = 0, .fptr = version, .flags = 0 }, - .{ .name = "db_open", .arity = 1, .fptr = db_open, .flags = 0 }, - .{ .name = "db_close", .arity = 1, .fptr = db_close, .flags = 0 }, - .{ .name = "txn_begin", .arity = 2, .fptr = txn_begin, .flags = 0 }, - .{ .name = "txn_commit", .arity = 1, .fptr = txn_commit, .flags = 0 }, - .{ .name = "txn_abort", .arity = 1, .fptr = txn_abort, .flags = 0 }, - .{ .name = "apply", .arity = 2, .fptr = apply, .flags = 0 }, - .{ .name = "schema", .arity = 1, .fptr = schema, .flags = 0 }, - .{ .name = "journal", .arity = 2, .fptr = journal, .flags = 0 }, -}; - -export fn nif_init(env: ?*beam.env, priv_data: [*c]?*anyopaque, load_info: beam.term) c_int { - _ = priv_data; - _ = load_info; - - // Check env is valid - if (env == null) { - return 1; - } - - // Register resource types - db_handle_type = beam.open_resource_type(env, "db_handle", null) catch { - return 1; - }; - - txn_handle_type = beam.open_resource_type(env, "txn_handle", null) catch { - return 1; - }; - - return 0; -} - -export const nif_entry = beam.ErlNifEntry{ - .major = beam.ERL_NIF_MAJOR_VERSION, - .minor = beam.ERL_NIF_MINOR_VERSION, - .name = "lith_nif", - .num_of_funcs = nif_funcs.len, - // SAFETY: nif_funcs is a file-level const array of ErlNifFunc structs with - // stable lifetime (static storage). The @constCast is required because the - // ErlNifEntry C ABI expects a mutable pointer, but the BEAM runtime only - // reads from this array. The @ptrCast converts [*]ErlNifFunc to [*c]const - // ErlNifFunc to match the C ABI pointer convention. - .funcs = @ptrCast(@constCast(&nif_funcs)), - .load = nif_init, - .reload = null, - .upgrade = null, - .unload = null, - .vm_variant = "beam.vanilla", - .options = 1, - .sizeof_ErlNifResourceTypeInit = @sizeOf(beam.ErlNifResourceTypeInit), -}; diff --git a/lithoglyph/beam/native/src/nif_helpers.c b/lithoglyph/beam/native/src/nif_helpers.c deleted file mode 100644 index b49711ca..00000000 --- a/lithoglyph/beam/native/src/nif_helpers.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Erlang NIF helpers - C wrappers for inline functions -// -// These wrappers allow Zig to call Erlang NIF inline functions -// without dealing with C varargs compatibility issues. - -#include - -// Wrapper for enif_make_tuple2 (inline function in erl_nif.h) -ERL_NIF_TERM nif_make_tuple2(ErlNifEnv* env, ERL_NIF_TERM t1, ERL_NIF_TERM t2) { - return enif_make_tuple2(env, t1, t2); -} - -// Wrapper for enif_make_tuple3 (inline function in erl_nif.h) -ERL_NIF_TERM nif_make_tuple3(ErlNifEnv* env, ERL_NIF_TERM t1, ERL_NIF_TERM t2, ERL_NIF_TERM t3) { - return enif_make_tuple3(env, t1, t2, t3); -} diff --git a/lithoglyph/beam/native_rust/Cargo.toml b/lithoglyph/beam/native_rust/Cargo.toml deleted file mode 100644 index 12dde354..00000000 --- a/lithoglyph/beam/native_rust/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -[package] -name = "lith_nif" -version = "1.0.0" -edition = "2021" -authors = ["Jonathan D.A. Jewell "] -description = "Lithoglyph BEAM NIF - Rust/Rustler bridge to the Lith C ABI" - -[lib] -name = "lith_nif" -crate-type = ["cdylib"] - -[dependencies] -rustler = "0.35" - -# Link against liblith (the core Zig FFI bridge). -# Set LITH_LIB_DIR to the directory containing liblith.so -# at build time, or place it in a standard library search path. -[package.metadata.lith] -link = "lith" diff --git a/lithoglyph/beam/native_rust/build.rs b/lithoglyph/beam/native_rust/build.rs deleted file mode 100644 index 9ebb4fe4..00000000 --- a/lithoglyph/beam/native_rust/build.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Build script for the Lithoglyph BEAM NIF (Rust/Rustler). -// -// Configures the linker to find liblith.so (the core Zig FFI bridge). -// The library path can be set via: -// - LITH_LIB_DIR environment variable -// - Default: ../../ffi/zig/zig-out/lib (relative to this crate) - -fn main() { - // Allow the user to override the library search path - let lib_dir = std::env::var("LITH_LIB_DIR").unwrap_or_else(|_| { - // Default to the Zig FFI output directory (relative to crate root) - let manifest_dir = - std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); - format!("{}/../../ffi/zig/zig-out/lib", manifest_dir) - }); - - println!("cargo:rustc-link-search=native={}", lib_dir); - println!("cargo:rustc-link-lib=dylib=lith"); - - // Re-run if the bridge header changes - println!("cargo:rerun-if-changed=../../generated/abi/bridge.h"); - println!("cargo:rerun-if-env-changed=LITH_LIB_DIR"); -} diff --git a/lithoglyph/beam/native_rust/src/lib.rs b/lithoglyph/beam/native_rust/src/lib.rs deleted file mode 100644 index 08a6489d..00000000 --- a/lithoglyph/beam/native_rust/src/lib.rs +++ /dev/null @@ -1,620 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Lithoglyph BEAM NIF - Rust/Rustler implementation -// -// This NIF connects BEAM (Erlang/Gleam/Elixir) to Lithoglyph via the Lith C ABI. -// Uses CBOR-encoded binaries for efficient data transfer. -// -// All database operations delegate to the core bridge (generated/abi/bridge.h) -// via FFI. Error codes from the bridge are translated to Elixir-style -// {:ok, result} / {:error, reason} tuples. - -use rustler::{Encoder, Env, ResourceArc, Term}; -use std::ptr; -use std::sync::Mutex; - -mod atoms { - rustler::atoms! { - ok, - error, - init_failed, - invalid_handle, - invalid_argument, - parse_failed, - validation_failed, - persist_failed, - alloc_failed, - resource_alloc_failed, - result_alloc_failed, - // Bridge status code atoms (match LithStatus enum in bridge.h) - internal_error, - not_found, - out_of_memory, - not_implemented, - txn_not_active, - txn_already_committed, - io_error, - corruption, - conflict, - already_exists, - unknown_error, - } -} - -// ============================================================ -// Lith C ABI declarations (from generated/abi/bridge.h) -// -// These extern functions are provided by liblith.so (the Zig FFI -// bridge implementation at ffi/zig/src/bridge.zig). The Rust NIF -// shared library links against liblith at load time. -// ============================================================ - -/// LithStatus codes (must match bridge.h LithStatus enum) -#[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[allow(dead_code)] -enum LithStatus { - Ok = 0, - ErrInternal = 1, - ErrNotFound = 2, - ErrInvalidArgument = 3, - ErrOutOfMemory = 4, - ErrNotImplemented = 5, - ErrTxnNotActive = 6, - ErrTxnAlreadyCommitted = 7, - ErrIoError = 8, - ErrCorruption = 9, - ErrConflict = 10, - ErrAlreadyExists = 11, -} - -impl LithStatus { - /// Convert a raw C int status code to an LithStatus enum variant. - fn from_raw(code: i32) -> Self { - match code { - 0 => Self::Ok, - 1 => Self::ErrInternal, - 2 => Self::ErrNotFound, - 3 => Self::ErrInvalidArgument, - 4 => Self::ErrOutOfMemory, - 5 => Self::ErrNotImplemented, - 6 => Self::ErrTxnNotActive, - 7 => Self::ErrTxnAlreadyCommitted, - 8 => Self::ErrIoError, - 9 => Self::ErrCorruption, - 10 => Self::ErrConflict, - 11 => Self::ErrAlreadyExists, - _ => Self::ErrInternal, - } - } - - /// Convert an LithStatus to a Rustler atom for BEAM error tuples. - fn to_atom(&self) -> rustler::Atom { - match self { - Self::Ok => atoms::ok(), - Self::ErrInternal => atoms::internal_error(), - Self::ErrNotFound => atoms::not_found(), - Self::ErrInvalidArgument => atoms::invalid_argument(), - Self::ErrOutOfMemory => atoms::out_of_memory(), - Self::ErrNotImplemented => atoms::not_implemented(), - Self::ErrTxnNotActive => atoms::txn_not_active(), - Self::ErrTxnAlreadyCommitted => atoms::txn_already_committed(), - Self::ErrIoError => atoms::io_error(), - Self::ErrCorruption => atoms::corruption(), - Self::ErrConflict => atoms::conflict(), - Self::ErrAlreadyExists => atoms::already_exists(), - } - } -} - -/// Opaque database handle from the core bridge -#[repr(C)] -struct LithDb { - _opaque: [u8; 0], -} - -/// Opaque transaction handle from the core bridge -#[repr(C)] -struct LithTxn { - _opaque: [u8; 0], -} - -/// Owned byte buffer passed across the FFI boundary -#[repr(C)] -#[derive(Debug, Clone, Copy)] -struct LgBlob { - ptr: *const u8, - len: usize, -} - -impl Default for LgBlob { - fn default() -> Self { - Self { - ptr: ptr::null(), - len: 0, - } - } -} - -/// Result type for operations returning data + provenance -#[repr(C)] -struct LgResult { - data: LgBlob, - provenance: LgBlob, - status: i32, - error_blob: LgBlob, -} - -/// Transaction mode -#[repr(C)] -#[derive(Clone, Copy)] -enum LgTxnMode { - ReadOnly = 0, - ReadWrite = 1, -} - -/// Render options for introspection functions -#[repr(C)] -#[derive(Clone, Copy)] -struct LgRenderOpts { - format: i32, - include_metadata: bool, -} - -extern "C" { - /// Open a Lith database. - fn lith_db_open( - path_ptr: *const u8, - path_len: usize, - opts_ptr: *const u8, - opts_len: usize, - out_db: *mut *mut LithDb, - out_err: *mut LgBlob, - ) -> i32; - - /// Close a Lith database and release resources. - fn lith_db_close(db: *mut LithDb) -> i32; - - /// Begin a new transaction. - fn lith_txn_begin( - db: *mut LithDb, - mode: LgTxnMode, - out_txn: *mut *mut LithTxn, - out_err: *mut LgBlob, - ) -> i32; - - /// Commit a transaction (6-phase WAL). - fn lith_txn_commit(txn: *mut LithTxn, out_err: *mut LgBlob) -> i32; - - /// Abort a transaction, discarding all buffered operations. - fn lith_txn_abort(txn: *mut LithTxn) -> i32; - - /// Apply an insert operation within a transaction. - fn lith_apply(txn: *mut LithTxn, op_ptr: *const u8, op_len: usize) -> LgResult; - - /// Get database schema information as JSON. - fn lith_introspect_schema( - db: *mut LithDb, - out_schema: *mut LgBlob, - out_err: *mut LgBlob, - ) -> i32; - - /// Render journal entries since a sequence number. - fn lith_render_journal( - db: *mut LithDb, - since: u64, - opts: LgRenderOpts, - out_text: *mut LgBlob, - out_err: *mut LgBlob, - ) -> i32; - - /// Free a blob allocated by the bridge. - fn lith_blob_free(blob: *mut LgBlob); - - /// Get Lith version as encoded integer (major * 10000 + minor * 100 + patch). - fn lith_version() -> u32; -} - -/// Helper: free an LgBlob if its pointer is non-null. -fn free_blob_if_nonnull(blob: &mut LgBlob) { - if !blob.ptr.is_null() { - // SAFETY: blob.ptr is non-null (checked above). The bridge allocated this - // blob and lith_blob_free is the designated deallocator for bridge-allocated - // blobs. After this call, blob.ptr is set to null by the bridge. - unsafe { lith_blob_free(blob) }; - } -} - -/// Helper: copy an LgBlob's data into a Vec, then free the blob. -fn blob_to_vec_and_free(blob: &mut LgBlob) -> Vec { - if blob.ptr.is_null() || blob.len == 0 { - free_blob_if_nonnull(blob); - return Vec::new(); - } - - // SAFETY: blob.ptr is non-null and blob.len > 0 (checked above). The bridge - // guarantees the pointer is valid for blob.len bytes until lith_blob_free() is - // called. We copy the data before freeing, so no use-after-free. - let data = unsafe { std::slice::from_raw_parts(blob.ptr, blob.len) }.to_vec(); - free_blob_if_nonnull(blob); - data -} - -// ============================================================ -// NIF Handle Wrappers -// ============================================================ - -/// Database handle wrapper — holds a raw LithDb pointer from the core bridge. -/// The pointer is protected by a Mutex because BEAM NIF resources can be accessed -/// from multiple scheduler threads. The Option allows us to close the database -/// once and set the pointer to None. -struct DbHandle { - lith: Mutex>, - #[allow(dead_code)] // Retained for debug logging and error messages - path: String, -} - -// SAFETY: LithDb is an opaque handle from the C bridge. The bridge implementation -// (ffi/zig/src/bridge.zig) uses thread-local state and global locks internally. -// We additionally protect the raw pointer with a Mutex to prevent concurrent -// access from multiple BEAM scheduler threads at the NIF level. -unsafe impl Send for DbHandle {} -unsafe impl Sync for DbHandle {} - -/// Transaction handle wrapper — holds a raw LithTxn pointer from the core bridge. -struct TxnHandle { - lith_txn: Mutex>, - #[allow(dead_code)] // Retained to prevent db from being dropped while txn is active - db: ResourceArc, - #[allow(dead_code)] // Retained for commit/abort validation - mode: TxnMode, -} - -// SAFETY: Same rationale as DbHandle — opaque handle protected by Mutex. -unsafe impl Send for TxnHandle {} -unsafe impl Sync for TxnHandle {} - -#[derive(Clone, Copy)] -enum TxnMode { - ReadOnly, - ReadWrite, -} - -impl TxnMode { - fn to_lg_mode(&self) -> LgTxnMode { - match self { - TxnMode::ReadOnly => LgTxnMode::ReadOnly, - TxnMode::ReadWrite => LgTxnMode::ReadWrite, - } - } -} - -// NIF functions -rustler::init!("lith_nif", load = load); - -#[allow(non_local_definitions)] -fn load(env: Env, _info: Term) -> bool { - let _ = rustler::resource!(DbHandle, env); - let _ = rustler::resource!(TxnHandle, env); - true -} - -/// Get Lithoglyph version from the core bridge. -/// The bridge encodes version as: major * 10000 + minor * 100 + patch. -/// Returns {Major, Minor, Patch} tuple. -#[rustler::nif] -fn version() -> (i32, i32, i32) { - // SAFETY: lith_version is a pure function with no side effects that - // returns a u32 version encoding. No pointers, no allocations. - let ver = unsafe { lith_version() }; - let major = (ver / 10000) as i32; - let minor = ((ver % 10000) / 100) as i32; - let patch = (ver % 100) as i32; - (major, minor, patch) -} - -/// Open a Lithoglyph database via the core bridge (lith_db_open). -/// Returns {:ok, DbRef} | {:error, reason} -#[rustler::nif] -fn db_open(path: String) -> Result, rustler::Atom> { - let mut out_db: *mut LithDb = ptr::null_mut(); - let mut out_err = LgBlob::default(); - - // SAFETY: path.as_ptr() and path.len() provide a valid byte slice for the - // duration of the call. opts_ptr is null with opts_len 0 (no options). - // out_db and out_err are valid mutable pointers to stack-allocated variables. - let status_code = unsafe { - lith_db_open( - path.as_ptr(), - path.len(), - ptr::null(), - 0, - &mut out_db, - &mut out_err, - ) - }; - - // Free error blob regardless of outcome - free_blob_if_nonnull(&mut out_err); - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok || out_db.is_null() { - return Err(atoms::init_failed()); - } - - let db = DbHandle { - lith: Mutex::new(Some(out_db)), - path, - }; - - Ok(ResourceArc::new(db)) -} - -/// Close a Lithoglyph database via the core bridge (lith_db_close). -/// Returns :ok | {:error, reason} -#[rustler::nif] -fn db_close(db: ResourceArc) -> Result { - let mut guard = db.lith.lock().map_err(|_| atoms::internal_error())?; - - match guard.take() { - Some(lith_ptr) => { - // SAFETY: lith_ptr was obtained from a successful lith_db_open call and - // has not been closed yet (we take() it from the Option to ensure - // single-close semantics). The pointer is valid for lith_db_close. - let status_code = unsafe { lith_db_close(lith_ptr) }; - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok { - return Err(status.to_atom()); - } - Ok(atoms::ok()) - } - None => { - // Already closed — idempotent - Ok(atoms::ok()) - } - } -} - -/// Begin a transaction via the core bridge (lith_txn_begin). -/// Returns {:ok, TxnRef} | {:error, reason} -#[rustler::nif] -fn txn_begin( - db: ResourceArc, - mode: String, -) -> Result, rustler::Atom> { - let txn_mode = match mode.as_str() { - "read_only" => TxnMode::ReadOnly, - "read_write" => TxnMode::ReadWrite, - _ => return Err(atoms::invalid_argument()), - }; - - let guard = db.lith.lock().map_err(|_| atoms::internal_error())?; - let lith_ptr = guard.ok_or_else(|| atoms::invalid_handle())?; - - let mut out_txn: *mut LithTxn = ptr::null_mut(); - let mut out_err = LgBlob::default(); - - // SAFETY: lith_ptr is a valid, non-null LithDb pointer obtained from a - // successful lith_db_open (checked via Option::ok_or above). out_txn and - // out_err are valid mutable pointers to stack-allocated variables. - let status_code = unsafe { - lith_txn_begin( - lith_ptr, - txn_mode.to_lg_mode(), - &mut out_txn, - &mut out_err, - ) - }; - - // Free error blob regardless of outcome - free_blob_if_nonnull(&mut out_err); - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok || out_txn.is_null() { - return Err(status.to_atom()); - } - - // Release the lock before creating the resource to avoid holding it - // longer than necessary. - drop(guard); - - let txn = TxnHandle { - lith_txn: Mutex::new(Some(out_txn)), - db, - mode: txn_mode, - }; - - Ok(ResourceArc::new(txn)) -} - -/// Commit a transaction via the core bridge (lith_txn_commit). -/// Executes the 6-phase WAL: journal -> sync -> blocks -> deletes -> superblock -> sync. -/// Returns :ok | {:error, reason} -#[rustler::nif] -fn txn_commit(txn: ResourceArc) -> Result { - let mut guard = txn.lith_txn.lock().map_err(|_| atoms::internal_error())?; - - match guard.take() { - Some(txn_ptr) => { - let mut out_err = LgBlob::default(); - - // SAFETY: txn_ptr was obtained from a successful lith_txn_begin call - // and has not been committed or aborted yet (we take() it from the - // Option to ensure single-use semantics). out_err is a valid mutable - // pointer to a stack-allocated LgBlob. - let status_code = unsafe { lith_txn_commit(txn_ptr, &mut out_err) }; - - free_blob_if_nonnull(&mut out_err); - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok { - return Err(status.to_atom()); - } - Ok(atoms::ok()) - } - None => { - // Transaction already committed or aborted - Err(atoms::txn_not_active()) - } - } -} - -/// Abort a transaction via the core bridge (lith_txn_abort). -/// Returns :ok | {:error, reason} -#[rustler::nif] -fn txn_abort(txn: ResourceArc) -> Result { - let mut guard = txn.lith_txn.lock().map_err(|_| atoms::internal_error())?; - - match guard.take() { - Some(txn_ptr) => { - // SAFETY: txn_ptr was obtained from a successful lith_txn_begin call - // and has not been committed or aborted yet (we take() it from the - // Option). The pointer is valid for lith_txn_abort. - let status_code = unsafe { lith_txn_abort(txn_ptr) }; - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok { - return Err(status.to_atom()); - } - Ok(atoms::ok()) - } - None => { - // Already aborted — idempotent - Ok(atoms::ok()) - } - } -} - -/// Apply an operation within a transaction via the core bridge (lith_apply). -/// The bridge parses the CBOR/JSON payload, validates it, and buffers the write. -/// -/// Returns {:ok, result_binary} | {:ok, result_binary, provenance_binary} | {:error, reason} -/// -/// NOTE: This NIF is registered as "apply" in the NIF table (see rustler::init! above -/// uses apply_op but the BEAM-facing name is "apply" via the #[rustler::nif(name = ...)] -/// attribute). -#[rustler::nif(name = "apply")] -fn apply_op<'a>(env: Env<'a>, txn: ResourceArc, op_cbor: rustler::Binary<'a>) -> Term<'a> { - if op_cbor.is_empty() || op_cbor.len() > 1_048_576 { - return (atoms::error(), atoms::invalid_argument()).encode(env); - } - - let guard = match txn.lith_txn.lock() { - Ok(g) => g, - Err(_) => return (atoms::error(), atoms::internal_error()).encode(env), - }; - - let txn_ptr = match *guard { - Some(ptr) => ptr, - None => return (atoms::error(), atoms::txn_not_active()).encode(env), - }; - - // SAFETY: txn_ptr is a valid, non-null LithTxn pointer obtained from a - // successful lith_txn_begin (checked via Option match above). op_cbor.as_ref() - // provides a valid byte slice for the duration of the call. - let mut result = unsafe { lith_apply(txn_ptr, op_cbor.as_ref().as_ptr(), op_cbor.len()) }; - - let status = LithStatus::from_raw(result.status); - if status != LithStatus::Ok { - free_blob_if_nonnull(&mut result.error_blob); - free_blob_if_nonnull(&mut result.data); - free_blob_if_nonnull(&mut result.provenance); - return (atoms::error(), status.to_atom()).encode(env); - } - - // Extract result data - let result_data = blob_to_vec_and_free(&mut result.data); - - // Include provenance token in response if the bridge provided one - let has_provenance = !result.provenance.ptr.is_null() && result.provenance.len > 0; - - if has_provenance { - let provenance_data = blob_to_vec_and_free(&mut result.provenance); - // Free error blob if any - free_blob_if_nonnull(&mut result.error_blob); - return (atoms::ok(), result_data, provenance_data).encode(env); - } - - // Free remaining blobs - free_blob_if_nonnull(&mut result.provenance); - free_blob_if_nonnull(&mut result.error_blob); - - (atoms::ok(), result_data).encode(env) -} - -/// Get database schema via the core bridge (lith_introspect_schema). -/// Returns {:ok, schema_json_binary} | {:error, reason} -#[rustler::nif] -fn schema(db: ResourceArc) -> Result, rustler::Atom> { - let guard = db.lith.lock().map_err(|_| atoms::internal_error())?; - let lith_ptr = guard.ok_or_else(|| atoms::invalid_handle())?; - - let mut out_schema = LgBlob::default(); - let mut out_err = LgBlob::default(); - - // SAFETY: lith_ptr is a valid, non-null LithDb pointer (checked via Option::ok_or - // above). out_schema and out_err are valid mutable pointers to stack-allocated - // LgBlob structs. - let status_code = unsafe { lith_introspect_schema(lith_ptr, &mut out_schema, &mut out_err) }; - - free_blob_if_nonnull(&mut out_err); - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok { - free_blob_if_nonnull(&mut out_schema); - return Err(status.to_atom()); - } - - let data = blob_to_vec_and_free(&mut out_schema); - - // If no schema data returned, provide an empty JSON object - if data.is_empty() { - return Ok(b"{}".to_vec()); - } - - Ok(data) -} - -/// Get journal entries since a sequence number via the core bridge (lith_render_journal). -/// Returns {:ok, journal_json_binary} | {:error, reason} -#[rustler::nif] -fn journal(db: ResourceArc, since: i64) -> Result, rustler::Atom> { - let guard = db.lith.lock().map_err(|_| atoms::internal_error())?; - let lith_ptr = guard.ok_or_else(|| atoms::invalid_handle())?; - - // Clamp negative values to 0 (meaning "from the beginning") - let since_u64 = if since >= 0 { since as u64 } else { 0 }; - - let opts = LgRenderOpts { - format: 0, // JSON - include_metadata: true, - }; - - let mut out_text = LgBlob::default(); - let mut out_err = LgBlob::default(); - - // SAFETY: lith_ptr is a valid, non-null LithDb pointer (checked via Option::ok_or - // above). opts is a valid LgRenderOpts value. out_text and out_err are valid - // mutable pointers to stack-allocated LgBlob structs. - let status_code = unsafe { - lith_render_journal(lith_ptr, since_u64, opts, &mut out_text, &mut out_err) - }; - - free_blob_if_nonnull(&mut out_err); - - let status = LithStatus::from_raw(status_code); - if status != LithStatus::Ok { - free_blob_if_nonnull(&mut out_text); - return Err(status.to_atom()); - } - - let data = blob_to_vec_and_free(&mut out_text); - - // If no journal data returned, provide an empty JSON array - if data.is_empty() { - return Ok(b"[]".to_vec()); - } - - Ok(data) -} diff --git a/lithoglyph/beam/src/lith_beam.gleam b/lithoglyph/beam/src/lith_beam.gleam deleted file mode 100644 index 719dd9f1..00000000 --- a/lithoglyph/beam/src/lith_beam.gleam +++ /dev/null @@ -1,7 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith BEAM - Main module - -/// Re-export client API -pub fn version() -> #(Int, Int, Int) { - lith_beam/client.version() -} diff --git a/lithoglyph/beam/src/lith_beam/client.gleam b/lithoglyph/beam/src/lith_beam/client.gleam deleted file mode 100644 index cc4b5e14..00000000 --- a/lithoglyph/beam/src/lith_beam/client.gleam +++ /dev/null @@ -1,255 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith client - Gleam interface to Lith via NIF - -import gleam/dynamic.{type Dynamic} -import gleam/erlang/atom.{type Atom} -import gleam/option.{type Option, None, Some} - -/// Lith database handle (opaque reference from NIF) -pub opaque type Connection { - Connection(ref: Dynamic) -} - -/// Lith transaction handle (opaque reference from NIF) -pub opaque type Transaction { - Transaction(ref: Dynamic, mode: TransactionMode) -} - -/// Transaction mode -pub type TransactionMode { - ReadOnly - ReadWrite -} - -/// Lith error types -pub type LithError { - ConnectionError(message: String) - TransactionError(message: String) - QueryError(message: String) - ValidationError(message: String) - ProvenanceError(message: String) - NotFound(entity: String, id: String) - PermissionDenied(action: String) - NifError(reason: Atom) - NifErrorWithData(reason: Atom, data: BitArray) -} - -/// Result type for Lith operations -pub type LithResult(a) = - Result(a, LithError) - -// ============================================================ -// External NIF Functions -// ============================================================ - -@external(erlang, "lith_nif", "version") -fn nif_version() -> #(Int, Int, Int) - -@external(erlang, "lith_nif", "db_open") -fn nif_db_open(path: BitArray) -> Dynamic - -@external(erlang, "lith_nif", "db_close") -fn nif_db_close(db: Dynamic) -> Dynamic - -@external(erlang, "lith_nif", "txn_begin") -fn nif_txn_begin(db: Dynamic, mode: Atom) -> Dynamic - -@external(erlang, "lith_nif", "txn_commit") -fn nif_txn_commit(txn: Dynamic) -> Dynamic - -@external(erlang, "lith_nif", "txn_abort") -fn nif_txn_abort(txn: Dynamic) -> Dynamic - -@external(erlang, "lith_nif", "apply") -fn nif_apply(txn: Dynamic, op: BitArray) -> Dynamic - -@external(erlang, "lith_nif", "schema") -fn nif_schema(db: Dynamic) -> Dynamic - -@external(erlang, "lith_nif", "journal") -fn nif_journal(db: Dynamic, since: Int) -> Dynamic - -// ============================================================ -// Helper Functions -// ============================================================ - -fn mode_to_atom(mode: TransactionMode) -> Atom { - case mode { - ReadOnly -> atom.create_from_string("read_only") - ReadWrite -> atom.create_from_string("read_write") - } -} - -fn decode_ok_ref(result: Dynamic) -> LithResult(Dynamic) { - case dynamic.tuple2(dynamic.dynamic, dynamic.dynamic)(result) { - Ok(#(tag, value)) -> { - case atom.from_dynamic(tag) { - Ok(a) if a == atom.create_from_string("ok") -> Ok(value) - Ok(a) -> Error(NifError(a)) - Error(_) -> Error(NifError(atom.create_from_string("decode_error"))) - } - } - Error(_) -> { - case atom.from_dynamic(result) { - Ok(a) if a == atom.create_from_string("ok") -> Ok(dynamic.from(Nil)) - Ok(a) -> Error(NifError(a)) - Error(_) -> Error(NifError(atom.create_from_string("decode_error"))) - } - } - } -} - -fn decode_ok_binary(result: Dynamic) -> LithResult(BitArray) { - case dynamic.tuple2(dynamic.dynamic, dynamic.bit_array)(result) { - Ok(#(tag, value)) -> { - case atom.from_dynamic(tag) { - Ok(a) if a == atom.create_from_string("ok") -> Ok(value) - Ok(a) -> Error(NifError(a)) - Error(_) -> Error(NifError(atom.create_from_string("decode_error"))) - } - } - Error(_) -> Error(NifError(atom.create_from_string("decode_error"))) - } -} - -fn decode_ok_binary_with_provenance( - result: Dynamic, -) -> LithResult(#(BitArray, Option(BitArray))) { - // Try tuple3 first (result with provenance) - case dynamic.tuple3(dynamic.dynamic, dynamic.bit_array, dynamic.bit_array)(result) { - Ok(#(tag, value, prov)) -> { - case atom.from_dynamic(tag) { - Ok(a) if a == atom.create_from_string("ok") -> Ok(#(value, Some(prov))) - Ok(a) -> Error(NifError(a)) - Error(_) -> Error(NifError(atom.create_from_string("decode_error"))) - } - } - Error(_) -> { - // Try tuple2 (result without provenance) - case decode_ok_binary(result) { - Ok(value) -> Ok(#(value, None)) - Error(e) -> Error(e) - } - } - } -} - -// ============================================================ -// Public API -// ============================================================ - -/// Get Lith version -pub fn version() -> #(Int, Int, Int) { - nif_version() -} - -/// Open a connection to a Lith database -pub fn connect(path: String) -> LithResult(Connection) { - let path_bits = <> - let result = nif_db_open(path_bits) - - case decode_ok_ref(result) { - Ok(ref) -> Ok(Connection(ref: ref)) - Error(e) -> Error(e) - } -} - -/// Close a Lith connection -pub fn disconnect(conn: Connection) -> LithResult(Nil) { - let Connection(ref: ref) = conn - let result = nif_db_close(ref) - - case decode_ok_ref(result) { - Ok(_) -> Ok(Nil) - Error(e) -> Error(e) - } -} - -/// Begin a transaction -pub fn begin_transaction( - conn: Connection, - mode: TransactionMode, -) -> LithResult(Transaction) { - let Connection(ref: db_ref) = conn - let result = nif_txn_begin(db_ref, mode_to_atom(mode)) - - case decode_ok_ref(result) { - Ok(txn_ref) -> Ok(Transaction(ref: txn_ref, mode: mode)) - Error(e) -> Error(e) - } -} - -/// Commit a transaction -pub fn commit(txn: Transaction) -> LithResult(Nil) { - let Transaction(ref: ref, ..) = txn - let result = nif_txn_commit(ref) - - case decode_ok_ref(result) { - Ok(_) -> Ok(Nil) - Error(e) -> Error(e) - } -} - -/// Abort a transaction -pub fn abort(txn: Transaction) -> LithResult(Nil) { - let Transaction(ref: ref, ..) = txn - let _ = nif_txn_abort(ref) - Ok(Nil) -} - -/// Apply an operation within a transaction -/// The operation should be CBOR-encoded -pub fn apply_operation( - txn: Transaction, - operation: BitArray, -) -> LithResult(#(BitArray, Option(BitArray))) { - let Transaction(ref: ref, ..) = txn - let result = nif_apply(ref, operation) - decode_ok_binary_with_provenance(result) -} - -/// Get database schema (CBOR-encoded) -pub fn get_schema(conn: Connection) -> LithResult(BitArray) { - let Connection(ref: ref) = conn - let result = nif_schema(ref) - decode_ok_binary(result) -} - -/// Get journal entries since a sequence number (CBOR-encoded) -pub fn get_journal(conn: Connection, since: Int) -> LithResult(BitArray) { - let Connection(ref: ref) = conn - let result = nif_journal(ref, since) - decode_ok_binary(result) -} - -// ============================================================ -// High-Level Operations -// ============================================================ - -/// Execute an operation in a transaction with automatic commit/abort -pub fn with_transaction( - conn: Connection, - mode: TransactionMode, - operation: fn(Transaction) -> LithResult(a), -) -> LithResult(a) { - case begin_transaction(conn, mode) { - Ok(txn) -> { - case operation(txn) { - Ok(result) -> { - case commit(txn) { - Ok(_) -> Ok(result) - Error(e) -> { - let _ = abort(txn) - Error(e) - } - } - } - Error(e) -> { - let _ = abort(txn) - Error(e) - } - } - } - Error(e) -> Error(e) - } -} diff --git a/lithoglyph/beam/src/lith_nif.erl b/lithoglyph/beam/src/lith_nif.erl deleted file mode 100644 index 7eae448a..00000000 --- a/lithoglyph/beam/src/lith_nif.erl +++ /dev/null @@ -1,31 +0,0 @@ -%% SPDX-License-Identifier: MPL-2.0 -%% Lith NIF module - Erlang wrapper - --module(lith_nif). --export([version/0, db_open/1, db_close/1, txn_begin/2, txn_commit/1, txn_abort/1, apply/2, schema/1, journal/2]). --on_load(init/0). - --define(NOT_LOADED, erlang:nif_error({not_loaded, ?MODULE})). - -%% Load the NIF -init() -> - PrivDir = case code:priv_dir(?MODULE) of - {error, _} -> - EbinDir = filename:dirname(code:which(?MODULE)), - AppPath = filename:dirname(EbinDir), - filename:join(AppPath, "priv"); - Path -> - Path - end, - erlang:load_nif(filename:join(PrivDir, "lith_nif"), 0). - -%% NIF function stubs (replaced when NIF loads) -version() -> ?NOT_LOADED. -db_open(_Path) -> ?NOT_LOADED. -db_close(_DbRef) -> ?NOT_LOADED. -txn_begin(_DbRef, _Mode) -> ?NOT_LOADED. -txn_commit(_TxnRef) -> ?NOT_LOADED. -txn_abort(_TxnRef) -> ?NOT_LOADED. -apply(_TxnRef, _OpCbor) -> ?NOT_LOADED. -schema(_DbRef) -> ?NOT_LOADED. -journal(_DbRef, _Since) -> ?NOT_LOADED. diff --git a/lithoglyph/beam/test/lith_beam_smoke_test.gleam b/lithoglyph/beam/test/lith_beam_smoke_test.gleam deleted file mode 100644 index fcbef6bd..00000000 --- a/lithoglyph/beam/test/lith_beam_smoke_test.gleam +++ /dev/null @@ -1,262 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// (PMPL-1.0-or-later preferred; MPL-2.0 required for Gleam ecosystem tooling) -// Author: Jonathan D.A. Jewell -// -// Lithoglyph BEAM smoke tests — Gleam gleeunit suite. -// -// Exercises the public Gleam client API (lith_beam/client.gleam) against the -// Lith NIF. These are smoke tests in the Testing & Benchmarking Taxonomy -// sense: they verify that the system is alive and functional end-to-end, -// not that every edge case is handled. -// -// Test scope (per taxonomy): -// - Version returns a structurally valid version triple. -// - connect / disconnect does not crash (graceful failure when NIF absent). -// - with_transaction: begin → apply → commit lifecycle. -// - Error handling: invalid DB reference is rejected cleanly. -// -// The NIF (.so) must be compiled and available in beam/priv/ for the -// connection/lifecycle tests to pass. If the NIF is absent, those tests -// will fail with a :function_clause or :undef error; the version test -// will still pass because it is linked statically via the Zig NIF shim. - -import gleam/io -import gleeunit -import gleeunit/should -import lith_beam/client.{ - type LithError, - ReadWrite, - connect, - disconnect, - get_journal, - get_schema, - version, - with_transaction, -} - -// =========================================================================== -// Test entry point -// =========================================================================== - -pub fn main() { - gleeunit.main() -} - -// =========================================================================== -// Smoke: version -// -// The version/0 function wraps the lith_nif:version/0 NIF call which is -// always linked (it does not depend on database files). It must return a -// three-element tuple of non-negative integers. -// =========================================================================== - -pub fn version_returns_triple_test() { - let #(major, minor, patch) = version() - - // Each component must be non-negative. - should.be_true(major >= 0) - should.be_true(minor >= 0) - should.be_true(patch >= 0) -} - -pub fn version_components_are_integers_test() { - // The Gleam type system ensures they are Int, but we verify the values - // are in a plausible range (not MAX_INT or negative). - let #(major, minor, patch) = version() - - should.be_true(major < 1_000) - should.be_true(minor < 1_000) - should.be_true(patch < 10_000) -} - -pub fn version_is_at_least_one_zero_zero_test() { - // Lithoglyph M10 PoC reports version 1.0.0. Enforce this as a baseline. - let #(major, _minor, _patch) = version() - should.be_true(major >= 1) -} - -// =========================================================================== -// Smoke: connect / disconnect (requires compiled NIF) -// -// These tests will fail gracefully (return an Error) if the NIF shared -// library is not compiled. The important thing is that the Gleam client -// returns a structured Result rather than crashing the BEAM process. -// =========================================================================== - -pub fn connect_to_temp_path_test() { - // Use a path in the OS temp directory for isolation. - let db_path = "/tmp/lith_gleam_smoke_test.lgh" - - let result = connect(db_path) - - case result { - Ok(conn) -> { - // NIF is available — verify we can close cleanly. - let close_result = disconnect(conn) - should.be_ok(close_result) - } - Error(_err) -> { - // NIF not compiled or path error — acceptable in CI without compiled NIF. - // The test verifies the client does not crash. - io.println("INFO: Lith NIF not available — connect smoke test skipped") - Nil - } - } -} - -pub fn connect_creates_new_database_test() { - // Each invocation uses a unique path to avoid state leakage between tests. - let db_path = "/tmp/lith_gleam_new_" <> int_to_string(erlang_unique_integer()) - - case connect(db_path) { - Ok(conn) -> { - let _ = disconnect(conn) - Nil - } - Error(_) -> { - io.println("INFO: Lith NIF unavailable — new database test skipped") - Nil - } - } -} - -// =========================================================================== -// Smoke: full lifecycle — connect → begin → apply → commit → disconnect -// =========================================================================== - -pub fn full_lifecycle_write_and_commit_test() { - let db_path = "/tmp/lith_gleam_lifecycle_" <> int_to_string(erlang_unique_integer()) - - case connect(db_path) { - Error(_) -> { - io.println("INFO: Lith NIF unavailable — lifecycle test skipped") - Nil - } - Ok(conn) -> { - // Minimal CBOR map: 0xA0 = empty map. - // A real operation would encode {"claim": "smoke test"}, but the - // empty map is the simplest valid CBOR document. - let cbor_op = <<0xA0>> - - let txn_result = - with_transaction(conn, ReadWrite, fn(txn) { - client.apply_operation(txn, cbor_op) - }) - - case txn_result { - Ok(#(_result_binary, _provenance)) -> { - // Apply succeeded — lifecycle is healthy. - should.be_ok(disconnect(conn)) - } - Error(_err) -> { - // Apply may fail if the Zig NIF reports an error for the empty map. - // This is acceptable for M10 PoC — the key thing is no crash. - let _ = disconnect(conn) - io.println("INFO: apply_operation returned error — acceptable for M10 PoC") - Nil - } - } - } - } -} - -pub fn schema_returns_cbor_binary_test() { - let db_path = "/tmp/lith_gleam_schema_" <> int_to_string(erlang_unique_integer()) - - case connect(db_path) { - Error(_) -> { - io.println("INFO: Lith NIF unavailable — schema test skipped") - Nil - } - Ok(conn) -> { - let schema_result = get_schema(conn) - - case schema_result do - Ok(cbor) -> { - // Schema must be at least 1 byte (the empty CBOR map 0xA0). - should.be_true(bit_size(cbor) >= 8) - let _ = disconnect(conn) - Nil - } - Error(_) -> { - let _ = disconnect(conn) - io.println("INFO: get_schema returned error — acceptable for M10 PoC") - Nil - } - end - } - } -} - -pub fn journal_returns_cbor_binary_test() { - let db_path = "/tmp/lith_gleam_journal_" <> int_to_string(erlang_unique_integer()) - - case connect(db_path) { - Error(_) -> { - io.println("INFO: Lith NIF unavailable — journal test skipped") - Nil - } - Ok(conn) -> { - // Request entries since sequence 0 (all entries). - let journal_result = get_journal(conn, 0) - - case journal_result { - Ok(cbor) -> { - // Journal must be at least 1 byte (the empty CBOR array 0x80). - should.be_true(bit_size(cbor) >= 8) - let _ = disconnect(conn) - Nil - } - Error(_) -> { - let _ = disconnect(conn) - io.println("INFO: get_journal returned error — acceptable for M10 PoC") - Nil - } - } - } - } -} - -// =========================================================================== -// Smoke: error handling -// =========================================================================== - -pub fn disconnect_invalid_conn_is_handled_test() { - // We cannot easily construct an invalid Connection (it is opaque), so - // we verify that connect to an invalid path returns a structured Error. - let bad_path = "/proc/this-is-read-only/lith.lgh" - - let result = connect(bad_path) - - // On Linux, /proc is read-only — connect must fail with Error, not crash. - case result { - Error(_err) -> { - // Correctly rejected — the error type is a structured LithError. - Nil - } - Ok(conn) -> { - // Unexpectedly succeeded — close it and move on. - let _ = disconnect(conn) - Nil - } - } -} - -// =========================================================================== -// Private helpers -// =========================================================================== - -// Call the Erlang unique_integer NIF to generate unique IDs for temp paths. -@external(erlang, "erlang", "unique_integer") -fn erlang_unique_integer() -> Int - -@external(erlang, "erlang", "integer_to_binary") -fn erlang_integer_to_binary(n: Int) -> BitArray - -fn int_to_string(n: Int) -> String { - let bits = erlang_integer_to_binary(n) - case bit_array.to_string(bits) { - Ok(s) -> s - Error(_) -> "unknown" - } -} diff --git a/lithoglyph/beam/test/lith_integration_test.exs b/lithoglyph/beam/test/lith_integration_test.exs deleted file mode 100644 index c5e4648b..00000000 --- a/lithoglyph/beam/test/lith_integration_test.exs +++ /dev/null @@ -1,443 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Integration tests for the Lith BEAM NIF pipeline. -# -# These tests exercise the full lifecycle of Lith operations through the -# NIF interface: open database -> begin transaction -> apply operations -> -# commit -> read back -> verify -> close. This validates that the Zig NIF -# (beam/native/src/lith_nif.zig) correctly bridges BEAM to the Lithoglyph -# storage engine via the Lith C ABI (generated/abi/bridge.h). -# -# The NIF uses CBOR-encoded binaries (RFC 8949) for data transfer, matching -# the encoding specification in spec/encoding.adoc. Lithoglyph-specific -# CBOR tags (39001-39008) are defined in core-zig/src/types.zig. -# -# Architecture: -# Elixir/ExUnit -> lith_nif.erl -> lith_nif.zig (NIF) -> Lith C ABI -# | -# core-zig/src/blocks.zig -# (4 KiB blocks, CRC32C) - -defmodule LithIntegrationTest do - use ExUnit.Case, async: false - - @test_db_dir System.tmp_dir!() - - # CBOR constants used in test payloads (from core-zig/src/cbor.zig) - # Map prefix: 0xA0-0xB7 for 0-23 pairs - # Text prefix: 0x60-0x77 for 0-23 byte strings - # Unsigned: 0x00-0x17 for 0-23, 0x18 for 1-byte arg, 0x19 for 2-byte arg - # Tag: 0xC0-0xD7 for 0-23, 0xD8 for 1-byte arg, 0xD9 for 2-byte arg - - setup do - db_path = Path.join(@test_db_dir, "lithoglyph_integration_#{:erlang.unique_integer([:positive])}.lgh") - - on_exit(fn -> - File.rm(db_path) - end) - - %{db_path: db_path} - end - - # ============================================================ - # Full Lifecycle: Insert -> Commit -> Read Back - # ============================================================ - - describe "full lifecycle: insert, commit, verify" do - test "open -> begin txn -> apply insert -> commit -> close", %{db_path: db_path} do - # Step 1: Open database - assert {:ok, db_ref} = :lith_nif.db_open(db_path) - refute is_nil(db_ref) - - # Step 2: Begin read-write transaction - assert {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - refute is_nil(txn_ref) - - # Step 3: Apply a document insert operation - # CBOR encoding of {"claim": "Integration test claim", "source": "ExUnit"} - # 0xA2 = map(2) - # 0x65 "claim" = text(5) "claim" - # 0x76 "Integration test claim" = text(22) "Integration test claim" - # 0x66 "source" = text(6) "source" - # 0x66 "ExUnit" = text(6) "ExUnit" - cbor_document = << - 0xA2, - 0x65, "claim", - 0x76, "Integration test claim", - 0x66, "source", - 0x66, "ExUnit" - >> - - result = :lith_nif.apply(txn_ref, cbor_document) - assert_apply_success(result) - - # Step 4: Commit the transaction - assert :ok = :lith_nif.txn_commit(txn_ref) - - # Step 5: Close the database - assert :ok = :lith_nif.db_close(db_ref) - end - - test "multiple inserts within a single transaction", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Insert document 1: {"claim": "GDP grew 2.1%", "source": "ONS"} - doc1 = <<0xA2, 0x65, "claim", 0x6E, "GDP grew 2.1%", 0x66, "source", 0x63, "ONS">> - result1 = :lith_nif.apply(txn_ref, doc1) - assert_apply_success(result1) - - # Insert document 2: {"claim": "CPI at 4.2%", "source": "ONS"} - doc2 = <<0xA2, 0x65, "claim", 0x6C, "CPI at 4.2%", 0x66, "source", 0x63, "ONS">> - result2 = :lith_nif.apply(txn_ref, doc2) - assert_apply_success(result2) - - # Insert document 3: {"claim": "Rates held at 5.25%", "source": "BoE"} - doc3 = <<0xA2, 0x65, "claim", 0x74, "Rates held at 5.25%", 0x66, "source", 0x63, "BoE">> - result3 = :lith_nif.apply(txn_ref, doc3) - assert_apply_success(result3) - - # Commit all three inserts atomically - assert :ok = :lith_nif.txn_commit(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "insert and verify block IDs are unique", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - doc1 = <<0xA1, 0x64, "data", 0x61, "1">> - doc2 = <<0xA1, 0x64, "data", 0x61, "2">> - - block_id_1 = extract_block_id(:lith_nif.apply(txn_ref, doc1)) - block_id_2 = extract_block_id(:lith_nif.apply(txn_ref, doc2)) - - # M10 PoC stub returns same block_id=1 for all inserts, but in production - # block IDs must be unique. This test documents the expected behaviour. - if block_id_1 != nil and block_id_2 != nil do - # When real storage is implemented, these must differ - assert is_integer(block_id_1) - assert is_integer(block_id_2) - end - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Transaction Abort / Rollback - # ============================================================ - - describe "transaction abort (rollback)" do - test "abort discards all pending operations", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Apply an insert - doc = <<0xA1, 0x65, "claim", 0x6C, "will be aborted">> - :lith_nif.apply(txn_ref, doc) - - # Abort instead of commit - assert :ok = :lith_nif.txn_abort(txn_ref) - - # Database should be unchanged (no committed data) - # Verify by checking schema/journal are still empty - assert {:ok, schema} = :lith_nif.schema(db_ref) - assert schema == <<0xA0>> - - :lith_nif.db_close(db_ref) - end - - test "abort after abort is idempotent", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - assert :ok = :lith_nif.txn_abort(txn_ref) - # Second abort should also return ok (idempotent per NIF implementation) - assert :ok = :lith_nif.txn_abort(txn_ref) - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Read-Only Transactions - # ============================================================ - - describe "read-only transactions" do - test "read-only transaction can read schema", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_only) - - # Read-only transaction should be able to observe current state - # Schema query goes through db_ref, not txn_ref, but txn context - # establishes a snapshot point - assert {:ok, schema} = :lith_nif.schema(db_ref) - assert is_binary(schema) - - :lith_nif.txn_commit(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "read-only transaction can read journal", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_only) - - assert {:ok, journal} = :lith_nif.journal(db_ref, 0) - assert is_binary(journal) - - :lith_nif.txn_commit(txn_ref) - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Database Reopen - # ============================================================ - - describe "database persistence across open/close" do - test "open, write, close, reopen succeeds", %{db_path: db_path} do - # First session: open and write - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - doc = <<0xA1, 0x65, "claim", 0x6E, "persistent data">> - :lith_nif.apply(txn_ref, doc) - :lith_nif.txn_commit(txn_ref) - :lith_nif.db_close(db_ref) - - # Second session: reopen the same database file - assert {:ok, db_ref2} = :lith_nif.db_open(db_path) - refute is_nil(db_ref2) - - # Should be able to read schema from reopened database - assert {:ok, _schema} = :lith_nif.schema(db_ref2) - - :lith_nif.db_close(db_ref2) - end - end - - # ============================================================ - # CBOR Payload Validation - # ============================================================ - - describe "CBOR payload handling" do - test "accepts minimal CBOR map (empty document)", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # 0xA0 = empty CBOR map {} - result = :lith_nif.apply(txn_ref, <<0xA0>>) - assert_apply_success(result) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "accepts CBOR with Lithoglyph PROMPT score tag (39006)", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # CBOR: {"score": tag(39006, 85)} - # 0xA1 = map(1) - # 0x65 "score" = text(5) "score" - # 0xD9 0x98 0x66 = tag(39006) [2-byte tag: 39006 = 0x9866] - # 0x18 0x55 = unsigned(85) [1-byte arg] - cbor_with_tag = <<0xA1, 0x65, "score", 0xD9, 0x98, 0x66, 0x18, 0x55>> - result = :lith_nif.apply(txn_ref, cbor_with_tag) - assert_apply_success(result) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "accepts CBOR with provenance tag (39004) and nested actor tag (39005)", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # CBOR: {"prov": tag(39004, {"actor": tag(39005, {"id": "u1", "type": "human"}), "rationale": "test"})} - # Simplified encoding: just the outer structure with tags - # tag(39004) = 0xD9 986C, tag(39005) = 0xD9 986D - cbor_with_provenance = << - 0xA1, - 0x64, "prov", - 0xD9, 0x98, 0x6C, - 0xA2, - 0x65, "actor", - 0xD9, 0x98, 0x6D, - 0xA2, - 0x62, "id", - 0x62, "u1", - 0x64, "type", - 0x65, "human", - 0x69, "rationale", - 0x64, "test" - >> - result = :lith_nif.apply(txn_ref, cbor_with_provenance) - assert_apply_success(result) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "rejects empty binary payload", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - assert {:error, :parse_failed} = :lith_nif.apply(txn_ref, <<>>) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "rejects payload exceeding 1 MiB limit", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # 1 MiB + 1 byte exceeds the lith_parse_cbor limit - oversized = :binary.copy(<<0xA0>>, 1_048_577) - assert {:error, :parse_failed} = :lith_nif.apply(txn_ref, oversized) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Concurrent Transactions (Sequential) - # ============================================================ - - describe "sequential transactions" do - test "multiple sequential transactions on same database", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - # Transaction 1: insert - {:ok, txn1} = :lith_nif.txn_begin(db_ref, :read_write) - doc1 = <<0xA1, 0x64, "data", 0x62, "t1">> - :lith_nif.apply(txn1, doc1) - assert :ok = :lith_nif.txn_commit(txn1) - - # Transaction 2: insert - {:ok, txn2} = :lith_nif.txn_begin(db_ref, :read_write) - doc2 = <<0xA1, 0x64, "data", 0x62, "t2">> - :lith_nif.apply(txn2, doc2) - assert :ok = :lith_nif.txn_commit(txn2) - - # Transaction 3: read-only - {:ok, txn3} = :lith_nif.txn_begin(db_ref, :read_only) - :lith_nif.txn_commit(txn3) - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Error Recovery - # ============================================================ - - describe "error recovery" do - test "database usable after failed apply", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Failed apply (empty payload) - {:error, _} = :lith_nif.apply(txn_ref, <<>>) - - # Abort the failed transaction - :lith_nif.txn_abort(txn_ref) - - # Start a new transaction - database should still be usable - {:ok, txn_ref2} = :lith_nif.txn_begin(db_ref, :read_write) - doc = <<0xA1, 0x64, "data", 0x62, "ok">> - result = :lith_nif.apply(txn_ref2, doc) - assert_apply_success(result) - - :lith_nif.txn_commit(txn_ref2) - :lith_nif.db_close(db_ref) - end - - test "database usable after aborted transaction", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - # Aborted transaction - {:ok, txn1} = :lith_nif.txn_begin(db_ref, :read_write) - doc = <<0xA1, 0x64, "data", 0x65, "abort">> - :lith_nif.apply(txn1, doc) - :lith_nif.txn_abort(txn1) - - # New transaction should work - {:ok, txn2} = :lith_nif.txn_begin(db_ref, :read_write) - doc2 = <<0xA1, 0x64, "data", 0x64, "good">> - result = :lith_nif.apply(txn2, doc2) - assert_apply_success(result) - :lith_nif.txn_commit(txn2) - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Full Evidence Document Lifecycle - # ============================================================ - - describe "evidence document lifecycle" do - test "insert evidence document with full CBOR structure", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Full evidence document matching spec/encoding.adoc example: - # {"claim": "Inflation at 10%", "source": "ONS", "score": tag(39006, 85)} - # CBOR encoding: - # A3 - map(3) - # 65 636C61696D - text(5) "claim" - # 70 496E666C... - text(16) "Inflation at 10%" - # 66 736F75726365 - text(6) "source" - # 63 4F4E53 - text(3) "ONS" - # 65 73636F7265 - text(5) "score" - # D9 9866 18 55 - tag(39006) unsigned(85) - evidence_cbor = << - 0xA3, - 0x65, "claim", - 0x70, "Inflation at 10%", - 0x66, "source", - 0x63, "ONS", - 0x65, "score", - 0xD9, 0x98, 0x66, 0x18, 0x55 - >> - - result = :lith_nif.apply(txn_ref, evidence_cbor) - assert_apply_success(result) - assert :ok = :lith_nif.txn_commit(txn_ref) - - # Verify journal has entries (will have content once real storage is active) - {:ok, journal} = :lith_nif.journal(db_ref, 0) - assert is_binary(journal) - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Helper Functions - # ============================================================ - - # Assert that an apply/2 result indicates success - defp assert_apply_success({:ok, result_binary}) do - assert is_binary(result_binary) - end - - defp assert_apply_success({:ok, result_binary, provenance_binary}) do - assert is_binary(result_binary) - assert is_binary(provenance_binary) - end - - defp assert_apply_success({:error, reason}) do - flunk("Expected apply to succeed, got error: #{inspect(reason)}") - end - - # Extract block ID from apply/2 result (8-byte big-endian unsigned integer) - defp extract_block_id({:ok, <>}), do: block_id - defp extract_block_id({:ok, <>, _prov}), do: block_id - defp extract_block_id(_), do: nil -end diff --git a/lithoglyph/beam/test/lith_nif_test.exs b/lithoglyph/beam/test/lith_nif_test.exs deleted file mode 100644 index 76d817c5..00000000 --- a/lithoglyph/beam/test/lith_nif_test.exs +++ /dev/null @@ -1,410 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# ExUnit tests for the Lith NIF interface. -# -# Tests the Erlang NIF functions exposed by lith_nif.erl, which delegates -# to the Zig NIF implementation in beam/native/src/lith_nif.zig. The NIF -# connects BEAM to the Lithoglyph storage engine via the Lith C ABI -# (generated/abi/bridge.h). -# -# NIF function signatures (from lith_nif.erl): -# version/0 -> {Major, Minor, Patch} -# db_open/1 -> {ok, DbRef} | {error, Reason} -# db_close/1 -> ok | {error, Reason} -# txn_begin/2 -> {ok, TxnRef} | {error, Reason} -# txn_commit/1 -> ok | {error, Reason} -# txn_abort/1 -> ok -# apply/2 -> {ok, ResultCbor} | {ok, ResultCbor, ProvCbor} | {error, Reason} -# schema/1 -> {ok, SchemaCbor} | {error, Reason} -# journal/2 -> {ok, JournalCbor} | {error, Reason} - -defmodule LithNifTest do - use ExUnit.Case, async: false - - # Temporary database path for tests. Each test creates a fresh database - # to avoid state leakage between tests. - @test_db_dir System.tmp_dir!() - - setup do - # Generate a unique database path for each test - db_path = Path.join(@test_db_dir, "lithoglyph_test_#{:erlang.unique_integer([:positive])}.lgh") - - on_exit(fn -> - # Clean up test database file after each test - File.rm(db_path) - end) - - %{db_path: db_path} - end - - # ============================================================ - # version/0 - # ============================================================ - - describe "version/0" do - test "returns a three-element tuple of non-negative integers" do - {major, minor, patch} = :lith_nif.version() - - assert is_integer(major) and major >= 0 - assert is_integer(minor) and minor >= 0 - assert is_integer(patch) and patch >= 0 - end - - test "returns version 1.0.0 for M10 PoC" do - assert {1, 0, 0} = :lith_nif.version() - end - end - - # ============================================================ - # db_open/1 - # ============================================================ - - describe "db_open/1" do - test "opens a new database and returns {ok, ref}", %{db_path: db_path} do - assert {:ok, db_ref} = :lith_nif.db_open(db_path) - assert is_reference(db_ref) - - # Clean up - assert :ok = :lith_nif.db_close(db_ref) - end - - test "returns a valid reference (not nil)", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - refute is_nil(db_ref) - - :lith_nif.db_close(db_ref) - end - - test "accepts binary path argument", %{db_path: db_path} do - # Path must be a binary (not a charlist) - assert {:ok, db_ref} = :lith_nif.db_open(db_path) - :lith_nif.db_close(db_ref) - end - - test "returns error for non-binary argument" do - # Passing an atom should trigger badarg from the NIF - assert_raise ArgumentError, fn -> - :lith_nif.db_open(:not_a_binary) - end - end - - test "returns error for integer argument" do - assert_raise ArgumentError, fn -> - :lith_nif.db_open(12345) - end - end - end - - # ============================================================ - # db_close/1 - # ============================================================ - - describe "db_close/1" do - test "closes an open database and returns ok", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - assert :ok = :lith_nif.db_close(db_ref) - end - - test "returns error for invalid handle" do - # Passing a non-resource reference should return an error tuple - result = :lith_nif.db_close(make_ref()) - assert {:error, :invalid_handle} = result - end - end - - # ============================================================ - # txn_begin/2 - # ============================================================ - - describe "txn_begin/2" do - test "begins a read_write transaction", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - assert is_reference(txn_ref) - - # Clean up - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "begins a read_only transaction", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_only) - assert is_reference(txn_ref) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns error for invalid mode atom", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - # Invalid mode should trigger badarg - assert_raise ArgumentError, fn -> - :lith_nif.txn_begin(db_ref, :invalid_mode) - end - - :lith_nif.db_close(db_ref) - end - - test "returns error for invalid database handle" do - result = :lith_nif.txn_begin(make_ref(), :read_write) - assert {:error, :invalid_handle} = result - end - - test "returns error when mode is not an atom", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert_raise ArgumentError, fn -> - :lith_nif.txn_begin(db_ref, "read_write") - end - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # txn_commit/1 - # ============================================================ - - describe "txn_commit/1" do - test "commits a read_write transaction", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - assert :ok = :lith_nif.txn_commit(txn_ref) - - :lith_nif.db_close(db_ref) - end - - test "returns error for invalid transaction handle" do - result = :lith_nif.txn_commit(make_ref()) - assert {:error, :invalid_handle} = result - end - end - - # ============================================================ - # txn_abort/1 - # ============================================================ - - describe "txn_abort/1" do - test "aborts a transaction and returns ok", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - assert :ok = :lith_nif.txn_abort(txn_ref) - - :lith_nif.db_close(db_ref) - end - - test "returns ok even for invalid handle (idempotent abort)" do - # Per NIF implementation: invalid handle returns :ok (already aborted) - assert :ok = :lith_nif.txn_abort(make_ref()) - end - end - - # ============================================================ - # apply/2 - # ============================================================ - - describe "apply/2" do - test "applies a CBOR-encoded insert operation", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Minimal CBOR map: {\"claim\": \"test\"} - # 0xA1 (map 1) + 0x65 \"claim\" + 0x64 \"test\" - cbor_op = <<0xA1, 0x65, "claim", 0x64, "test">> - - result = :lith_nif.apply(txn_ref, cbor_op) - - case result do - {:ok, result_binary} -> - # Result should be a binary containing the block ID (8 bytes big-endian) - assert is_binary(result_binary) - assert byte_size(result_binary) == 8 - - {:ok, result_binary, provenance_binary} -> - # Alternate form with provenance - assert is_binary(result_binary) - assert is_binary(provenance_binary) - - {:error, reason} -> - # M10 PoC may return errors for certain inputs - assert is_atom(reason) - end - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns error for empty CBOR binary", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Empty binary should fail CBOR parsing (cbor_len == 0 check in NIF) - assert {:error, :parse_failed} = :lith_nif.apply(txn_ref, <<>>) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns error for oversized CBOR binary", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Exceeds 1 MiB limit in lith_parse_cbor (cbor_len > 1048576) - oversized = :binary.copy(<<0>>, 1_048_577) - assert {:error, :parse_failed} = :lith_nif.apply(txn_ref, oversized) - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns error for non-binary operation argument", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - assert_raise ArgumentError, fn -> - :lith_nif.apply(txn_ref, :not_a_binary) - end - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns error for invalid transaction handle" do - cbor_op = <<0xA0>> - result = :lith_nif.apply(make_ref(), cbor_op) - assert {:error, :invalid_handle} = result - end - - test "applies a CBOR empty map operation", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # Empty CBOR map (0xA0) - minimal valid CBOR document - cbor_op = <<0xA0>> - result = :lith_nif.apply(txn_ref, cbor_op) - - # Should succeed (valid CBOR, even if minimal) - case result do - {:ok, _} -> :ok - {:ok, _, _} -> :ok - {:error, reason} -> assert is_atom(reason) - end - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - - test "returns 8-byte block ID on successful apply", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - {:ok, txn_ref} = :lith_nif.txn_begin(db_ref, :read_write) - - # CBOR map with single field: {"data": "value"} - cbor_op = <<0xA1, 0x64, "data", 0x65, "value">> - - case :lith_nif.apply(txn_ref, cbor_op) do - {:ok, result_binary} -> - # Block ID is returned as 8-byte big-endian unsigned integer - assert byte_size(result_binary) == 8 - <> = result_binary - # M10 PoC returns block_id=1 (from lith_persist stub) - assert block_id >= 1 - - {:ok, result_binary, _provenance} -> - assert byte_size(result_binary) == 8 - - {:error, _} -> - :ok - end - - :lith_nif.txn_abort(txn_ref) - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # schema/1 - # ============================================================ - - describe "schema/1" do - test "returns schema as CBOR binary", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert {:ok, schema_cbor} = :lith_nif.schema(db_ref) - assert is_binary(schema_cbor) - - # M10 PoC returns empty CBOR map (0xA0) - assert schema_cbor == <<0xA0>> - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # journal/2 - # ============================================================ - - describe "journal/2" do - test "returns journal entries as CBOR binary", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert {:ok, journal_cbor} = :lith_nif.journal(db_ref, 0) - assert is_binary(journal_cbor) - - # M10 PoC returns empty CBOR array (0x80) - assert journal_cbor == <<0x80>> - - :lith_nif.db_close(db_ref) - end - - test "accepts non-zero since parameter", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - # Request entries since sequence 100 - assert {:ok, journal_cbor} = :lith_nif.journal(db_ref, 100) - assert is_binary(journal_cbor) - - :lith_nif.db_close(db_ref) - end - end - - # ============================================================ - # Arity validation - # ============================================================ - - describe "arity validation" do - test "db_open requires exactly 1 argument" do - assert_raise UndefinedFunctionError, fn -> - :lith_nif.db_open() - end - end - - test "txn_begin requires exactly 2 arguments", %{db_path: db_path} do - {:ok, db_ref} = :lith_nif.db_open(db_path) - - assert_raise UndefinedFunctionError, fn -> - :lith_nif.txn_begin(db_ref) - end - - :lith_nif.db_close(db_ref) - end - - test "apply requires exactly 2 arguments" do - assert_raise UndefinedFunctionError, fn -> - :lith_nif.apply(make_ref()) - end - end - - test "journal requires exactly 2 arguments" do - assert_raise UndefinedFunctionError, fn -> - :lith_nif.journal(make_ref()) - end - end - end -end diff --git a/lithoglyph/beam/test/test_helper.exs b/lithoglyph/beam/test/test_helper.exs deleted file mode 100644 index 0fbf255d..00000000 --- a/lithoglyph/beam/test/test_helper.exs +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# ExUnit test helper for Lithoglyph BEAM integration tests. -# Starts ExUnit and configures the test environment for Lith NIF testing. -# -# The NIF shared library (lith_nif.so) must be compiled and placed in -# the priv/ directory before running tests. See beam/BUILD-STATUS.md -# for build instructions. - -ExUnit.start( - capture_log: true, - exclude: [:skip, :wip], - formatters: [ExUnit.CLIFormatter] -) diff --git a/lithoglyph/beam/test_minimal.erl b/lithoglyph/beam/test_minimal.erl deleted file mode 100644 index 08f2fc45..00000000 --- a/lithoglyph/beam/test_minimal.erl +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env escript -%% SPDX-License-Identifier: MPL-2.0 -%% Minimal NIF test - just version() - -main(_) -> - io:format("~n=== Minimal Lith NIF Test ===~n~n"), - - % Test 1: Load NIF - io:format("Test 1: Loading NIF...~n"), - case erlang:load_nif("./priv/lith_nif", 0) of - ok -> - io:format(" ✓ NIF loaded successfully~n~n"); - {error, {Reason, Text}} -> - io:format(" ✗ NIF load failed: ~p - ~s~n", [Reason, Text]), - halt(1) - end, - - % Test 2: Version - io:format("Test 2: Calling version()...~n"), - Version = lith_nif:version(), - io:format(" ✓ Version: ~p~n~n", [Version]), - - io:format("=== Test passed! ===~n~n"), - halt(0). diff --git a/lithoglyph/beam/test_nif.erl b/lithoglyph/beam/test_nif.erl deleted file mode 100644 index ac981f87..00000000 --- a/lithoglyph/beam/test_nif.erl +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env escript -%% SPDX-License-Identifier: MPL-2.0 -%% Quick test of Lith NIF - -main(_) -> - io:format("~n=== Lith-BEAM NIF Test ===~n~n"), - - % Add priv to library path - code:add_pathz("priv"), - - % Test 1: Load NIF - io:format("Test 1: Loading NIF...~n"), - case erlang:load_nif("./priv/lith_nif", 0) of - ok -> - io:format(" ✓ NIF loaded successfully~n~n"); - {error, {Reason, Text}} -> - io:format(" ✗ NIF load failed: ~p - ~s~n", [Reason, Text]), - halt(1) - end, - - % Test 2: Version - io:format("Test 2: Calling version()...~n"), - try lith_nif:version() of - {Major, Minor, Patch} -> - io:format(" ✓ Version: ~p.~p.~p~n~n", [Major, Minor, Patch]) - catch - error:VersionReason -> - io:format(" ✗ Version failed: ~p~n", [VersionReason]), - halt(1) - end, - - % Test 3: Open database - io:format("Test 3: Opening database...~n"), - DbPath = <<"/tmp/lith_test">>, - try lith_nif:db_open(DbPath) of - {ok, DbRef} -> - io:format(" ✓ Database opened: ~p~n", [DbRef]), - - % Test 4: Begin transaction - io:format("~nTest 4: Beginning transaction...~n"), - try lith_nif:txn_begin(DbRef, read_write) of - {ok, TxnRef} -> - io:format(" ✓ Transaction started: ~p~n", [TxnRef]), - - % Test 5: Commit transaction - io:format("~nTest 5: Committing transaction...~n"), - try lith_nif:txn_commit(TxnRef) of - ok -> - io:format(" ✓ Transaction committed~n"); - CommitError -> - io:format(" ✗ Commit failed: ~p~n", [CommitError]) - catch - error:ErrCommit -> - io:format(" ✗ Commit error: ~p~n", [ErrCommit]) - end; - TxnError -> - io:format(" ✗ Transaction failed: ~p~n", [TxnError]) - catch - error:ErrTxn -> - io:format(" ✗ Transaction error: ~p~n", [ErrTxn]) - end, - - % Test 6: Close database - io:format("~nTest 6: Closing database...~n"), - try lith_nif:db_close(DbRef) of - ok -> - io:format(" ✓ Database closed~n"); - CloseError -> - io:format(" ✗ Close failed: ~p~n", [CloseError]) - catch - error:ErrClose -> - io:format(" ✗ Close error: ~p~n", [ErrClose]) - end; - OpenError -> - io:format(" ✗ Open failed: ~p~n", [OpenError]), - halt(1) - catch - error:ErrOpen -> - io:format(" ✗ Open error: ~p~n", [ErrOpen]), - halt(1) - end, - - io:format("~n=== All tests passed! ===~n~n"), - halt(0). diff --git a/lithoglyph/beam/test_rust.erl b/lithoglyph/beam/test_rust.erl deleted file mode 100755 index 1cbd5068..00000000 --- a/lithoglyph/beam/test_rust.erl +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env escript -%% SPDX-License-Identifier: MPL-2.0 -%% Test Lith Rust NIF - -main(_) -> - io:format("~n=== Lith Rust NIF Test ===~n~n"), - - % Add paths - true = code:add_patha("ebin"), - - % Test 1: Version - io:format("Test 1: Calling version()...~n"), - Version = lith_nif:version(), - io:format(" ✓ Version: ~p~n~n", [Version]), - - % Test 2: Open database - io:format("Test 2: Opening database...~n"), - DbRef = lith_nif:db_open(<<"/tmp/lith_test">>), - io:format(" ✓ Database opened~n~n", []), - - % Test 3: Begin transaction - io:format("Test 3: Beginning transaction...~n"), - {ok, TxnRef} = lith_nif:txn_begin(DbRef, <<"read_write">>), - io:format(" ✓ Transaction started~n~n", []), - - % Test 4: Apply operation (with CBOR map) - io:format("Test 4: Applying operation...~n"), - CborMap = <<16#a1, 16#01, 16#02>>, % CBOR map {1: 2} - {ok, BlockId} = lith_nif:apply(TxnRef, CborMap), - io:format(" ✓ Operation applied, block ID: ~p~n~n", [BlockId]), - - % Test 5: Commit transaction - io:format("Test 5: Committing transaction...~n"), - ok = lith_nif:txn_commit(TxnRef), - io:format(" ✓ Transaction committed~n~n", []), - - % Test 6: Schema - io:format("Test 6: Getting schema...~n"), - Schema = lith_nif:schema(DbRef), - io:format(" ✓ Schema: ~p~n~n", [Schema]), - - % Test 7: Journal - io:format("Test 7: Getting journal...~n"), - Journal = lith_nif:journal(DbRef, 0), - io:format(" ✓ Journal: ~p~n~n", [Journal]), - - % Test 8: Close database - io:format("Test 8: Closing database...~n"), - ok = lith_nif:db_close(DbRef), - io:format(" ✓ Database closed~n~n", []), - - io:format("=== All tests passed! ===~n~n"), - halt(0). diff --git a/lithoglyph/clients/README.md b/lithoglyph/clients/README.md deleted file mode 100644 index 31176a67..00000000 --- a/lithoglyph/clients/README.md +++ /dev/null @@ -1,258 +0,0 @@ -# Lith Client Libraries - -Official client libraries for Lith - the narrative-first, reversible, audit-grade database. - -## Available Clients - -| Language | Package | Status | Features | -|----------|---------|--------|----------| -| ReScript | `@lith/rescript` | Stable | Type-safe, Deno runtime | -| PHP | `lith/lith-php` | Stable | PSR-18, Laravel/Symfony | - -## Quick Start - -### ReScript (Deno) - -```rescript -open Lith -open Lith_Types - -let client = make(~baseUrl="http://localhost:8080") - -// Query with fluent builder -let result = await client->queryWith( - select() - ->from("articles") - ->whereField("status", Eq, "published") - ->limit(10) - ->withProvenance("user@example.com", "List articles") -) - -result.rows->Array.forEach(row => Console.log(row)) -``` - -### PHP - -```php -use Lith\LithClient; -use Lith\Query\CompareOp; - -$client = LithClient::fromEnv($httpClient, $httpFactory, $httpFactory); - -$result = $client->queryWith( - $client->select() - ->from('articles') - ->whereField('status', CompareOp::EQ, 'published') - ->limit(10) - ->withProvenance('user@example.com', 'List articles') -); - -foreach ($result->rows as $row) { - echo $row['title'] . "\n"; -} -``` - -## Features - -All clients support: - -- **GQL Queries** - Full query language support -- **Query Builder** - Type-safe, fluent query construction -- **Provenance** - Audit trail metadata on all operations -- **Collections** - List, create, delete collections -- **Journal** - Access audit log entries -- **Normalization** - Discover dependencies, analyze normal forms -- **Migration** - Start and commit schema migrations -- **Health** - Server health checks - -## Authentication - -```rescript -// ReScript - API Key -let client = make(~baseUrl="http://localhost:8080", ~apiKey="your-api-key") - -// ReScript - Bearer Token -let client = make(~baseUrl="http://localhost:8080", ~bearerToken="your-jwt") - -// ReScript - From Environment -let client = fromEnv() // Uses LITH_URL, LITH_API_KEY -``` - -```php -// PHP - API Key -$client->setApiKey('your-api-key'); - -// PHP - Bearer Token -$client->setBearerToken('your-jwt'); - -// PHP - From Environment -$client = LithClient::fromEnv(...); // Uses LITH_URL, LITH_API_KEY -``` - -## Query Builder - -### SELECT - -```rescript -// ReScript -select() -->from("users") -->select(["id", "name", "email"]) -->whereField("active", Eq, true) -->whereField("created_at", Gt, "2025-01-01") -->orderBy("name", ~ascending=true) -->limit(100) -->offset(0) -``` - -```php -// PHP -$client->select() - ->from('users') - ->select(['id', 'name', 'email']) - ->whereField('active', CompareOp::EQ, true) - ->whereField('created_at', CompareOp::GT, '2025-01-01') - ->orderBy('name', ascending: true) - ->limit(100) - ->offset(0); -``` - -### INSERT - -```rescript -// ReScript -insert() -->into("users") -->values({"name": "Alice", "email": "alice@example.com"}) -->withProvenance("admin", "Create user") -``` - -```php -// PHP -$client->insert() - ->into('users') - ->values(['name' => 'Alice', 'email' => 'alice@example.com']) - ->withProvenance('admin', 'Create user'); -``` - -### UPDATE - -```rescript -// ReScript -update() -->collection("users") -->set("status", "inactive") -->where(Field("last_login", Lt, "2024-01-01")) -->withProvenance("cleanup-job", "Deactivate old users") -``` - -```php -// PHP -$client->update() - ->collection('users') - ->set('status', 'inactive') - ->where(new FieldFilter('last_login', CompareOp::LT, '2024-01-01')) - ->withProvenance('cleanup-job', 'Deactivate old users'); -``` - -### DELETE - -```rescript -// ReScript -delete() -->from("sessions") -->where(Field("expires_at", Lt, now())) -->withProvenance("session-cleanup", "Remove expired sessions") -``` - -```php -// PHP -$client->delete() - ->from('sessions') - ->where(new FieldFilter('expires_at', CompareOp::LT, $now)) - ->withProvenance('session-cleanup', 'Remove expired sessions'); -``` - -## Filter Expressions - -Complex filters with AND, OR, NOT: - -```rescript -// ReScript -let filter = And( - Field("status", Eq, "active"), - Or( - Field("role", Eq, "admin"), - Field("role", Eq, "moderator") - ) -) - -select()->from("users")->where(filter) -``` - -```php -// PHP -$filter = new AndFilter( - new FieldFilter('status', CompareOp::EQ, 'active'), - new OrFilter( - new FieldFilter('role', CompareOp::EQ, 'admin'), - new FieldFilter('role', CompareOp::EQ, 'moderator') - ) -); - -$client->select()->from('users')->where($filter); -``` - -## Error Handling - -```rescript -// ReScript -try { - let result = await client->query("SELECT * FROM missing") - // ... handle result -} catch { -| LithError(err) => Console.error(`Error: ${err.message}`) -} -``` - -```php -// PHP -try { - $result = $client->query('SELECT * FROM missing'); -} catch (LithException $e) { - echo "Error {$e->code}: {$e->getMessage()}\n"; -} -``` - -## Directory Structure - -``` -clients/ -├── README.md # This file -├── rescript/ # ReScript client -│ ├── src/ -│ │ ├── Lith.res -│ │ ├── Lith_Types.res -│ │ └── Lith_Query.res -│ ├── rescript.json -│ └── deno.json -└── php/ # PHP client - ├── src/ - │ ├── LithClient.php - │ ├── Types/ - │ └── Query/ - ├── composer.json - └── README.md -``` - -## SDK Generator - -The `tools/sdk-gen/` directory contains a code generator that can produce basic client skeletons from the API specification. The hand-crafted clients in this directory provide better ergonomics and are recommended for production use. - -## Contributing - -See the main Lith repository for contribution guidelines. - -## License - -PMPL-1.0-or-later diff --git a/lithoglyph/clients/php/README.md b/lithoglyph/clients/php/README.md deleted file mode 100644 index 46d6af57..00000000 --- a/lithoglyph/clients/php/README.md +++ /dev/null @@ -1,372 +0,0 @@ -# lith/lith-php - -PHP client for Lith - the narrative-first, reversible, audit-grade database. - -## Requirements - -- PHP 8.1 or higher -- PSR-18 HTTP Client (e.g., Guzzle) -- PSR-17 HTTP Factories - -## Installation - -```bash -composer require lith/lith-php -``` - -## Quick Start - -```php -health(); -echo "Server is " . $health->status->value . "\n"; -``` - -## Query Examples - -### Using Query Builder - -```php -use Lith\Query\CompareOp; - -// Build a SELECT query -$result = $lith->queryWith( - $lith->select() - ->from('articles') - ->select(['id', 'title', 'author']) - ->whereField('status', CompareOp::EQ, 'published') - ->whereField('views', CompareOp::GT, 100) - ->orderBy('createdAt', ascending: false) - ->limit(10) - ->withProvenance('editor@news.org', 'Daily review of popular articles') -); - -foreach ($result->rows as $row) { - echo $row['title'] . "\n"; -} -``` - -### Raw GQL - -```php -$result = $lith->query( - 'SELECT * FROM articles WHERE status = "published" LIMIT 10', - new Provenance('editor@news.org', 'Daily review') -); - -echo "Found {$result->rowCount} articles\n"; -``` - -### Insert - -```php -$result = $lith->insertWith( - $lith->insert() - ->into('articles') - ->values([ - 'title' => 'Breaking News', - 'author' => 'reporter@news.org', - 'content' => '...', - ]) - ->withProvenance('reporter@news.org', 'New article submission') -); -``` - -### Update - -```php -use Lith\Query\FieldFilter; -use Lith\Query\CompareOp; - -$result = $lith->updateWith( - $lith->update() - ->collection('articles') - ->set('status', 'archived') - ->where(new FieldFilter('createdAt', CompareOp::LT, '2025-01-01')) - ->withProvenance('admin@news.org', 'Archive old articles') -); -``` - -### Delete - -```php -$result = $lith->deleteWith( - $lith->delete() - ->from('drafts') - ->where(new FieldFilter('status', CompareOp::EQ, 'abandoned')) - ->withProvenance('cleanup@news.org', 'Remove abandoned drafts') -); -``` - -## Collection Operations - -```php -use Lith\Types\CollectionType; - -// List collections -$collections = $lith->listCollections(); -foreach ($collections as $col) { - echo "{$col->name}: {$col->documentCount} documents\n"; -} - -// Create collection -$newCol = $lith->createCollection( - 'users', - CollectionType::DOCUMENT, - [ - 'type' => 'object', - 'properties' => [ - 'email' => ['type' => 'string'], - 'name' => ['type' => 'string'], - ], - ] -); - -// Get collection -$col = $lith->getCollection('articles'); - -// Delete collection -$lith->deleteCollection('temp_data'); -``` - -## Journal Operations - -```php -// Get recent journal entries -$entries = $lith->getJournal( - since: 1000, - limit: 50, - collection: 'articles' -); - -foreach ($entries as $entry) { - echo "[{$entry->seq}] {$entry->operation->value} on {$entry->collection}\n"; -} -``` - -## Normalization - -```php -use Lith\Types\NormalForm; - -// Discover functional dependencies -$fds = $lith->discoverDependencies('orders', confidence: 0.95); - -foreach ($fds as $fd) { - $det = implode(', ', $fd->determinant); - echo "{$det} -> {$fd->dependent} ({$fd->confidence->value})\n"; -} - -// Analyze normal form -$analysis = $lith->analyzeNormalForm('orders'); - -echo "Current: {$analysis->currentForm->value}, Target: {$analysis->targetForm->value}\n"; - -foreach ($analysis->violations as $violation) { - echo "Violation: {$violation}\n"; -} - -foreach ($analysis->recommendations as $rec) { - echo "Recommendation: {$rec}\n"; -} -``` - -## Migration - -```php -// Start migration to BCNF -$migration = $lith->startMigration('orders', NormalForm::BCNF); - -echo "Migration {$migration->id} started: {$migration->narrative}\n"; - -// When ready, commit -$lith->commitMigration($migration->id); -``` - -## Authentication - -```php -// API Key -$lith = new LithClient('http://localhost:8080', $httpClient, $httpFactory, $httpFactory); -$lith->setApiKey('your-api-key'); - -// Bearer Token (JWT) -$lith->setBearerToken('your-jwt-token'); -``` - -## Filter Expressions - -The query builder supports complex filter expressions: - -```php -use Lith\Query\FieldFilter; -use Lith\Query\AndFilter; -use Lith\Query\OrFilter; -use Lith\Query\NotFilter; -use Lith\Query\CompareOp; - -// Simple comparison -$filter = new FieldFilter('status', CompareOp::EQ, 'active'); - -// AND -$filter = new AndFilter( - new FieldFilter('status', CompareOp::EQ, 'active'), - new FieldFilter('views', CompareOp::GT, 100) -); - -// OR -$filter = new OrFilter( - new FieldFilter('priority', CompareOp::EQ, 'high'), - new FieldFilter('urgent', CompareOp::EQ, true) -); - -// NOT -$filter = new NotFilter(new FieldFilter('deleted', CompareOp::EQ, true)); - -// Complex nested -$filter = new AndFilter( - new FieldFilter('status', CompareOp::EQ, 'published'), - new OrFilter( - new FieldFilter('category', CompareOp::EQ, 'news'), - new FieldFilter('featured', CompareOp::EQ, true) - ) -); - -// Use in query -$result = $lith->queryWith( - $lith->select() - ->from('articles') - ->where($filter) -); -``` - -## Error Handling - -All API methods throw `LithException` on error: - -```php -use Lith\LithException; - -try { - $result = $lith->query('SELECT * FROM articles'); -} catch (LithException $e) { - echo "Error {$e->code}: {$e->getMessage()}\n"; - if ($e->details) { - print_r($e->details); - } -} -``` - -## Laravel Integration - -```php -// config/services.php -return [ - 'lith' => [ - 'url' => env('LITH_URL', 'http://localhost:8080'), - 'api_key' => env('LITH_API_KEY'), - ], -]; - -// app/Providers/AppServiceProvider.php -use Lith\LithClient; -use GuzzleHttp\Client; -use GuzzleHttp\Psr7\HttpFactory; - -public function register(): void -{ - $this->app->singleton(LithClient::class, function ($app) { - $httpClient = new Client(); - $httpFactory = new HttpFactory(); - - $client = new LithClient( - config('services.lith.url'), - $httpClient, - $httpFactory, - $httpFactory - ); - - if ($apiKey = config('services.lith.api_key')) { - $client->setApiKey($apiKey); - } - - return $client; - }); -} - -// In a controller -public function index(LithClient $lith) -{ - $articles = $lith->queryWith( - $lith->select() - ->from('articles') - ->whereField('status', '=', 'published') - ->limit(10) - ); - - return view('articles.index', ['articles' => $articles->rows]); -} -``` - -## Symfony Integration - -```yaml -# config/services.yaml -services: - GuzzleHttp\Client: ~ - GuzzleHttp\Psr7\HttpFactory: ~ - - Lith\LithClient: - arguments: - $baseUrl: '%env(LITH_URL)%' - $httpClient: '@GuzzleHttp\Client' - $requestFactory: '@GuzzleHttp\Psr7\HttpFactory' - $streamFactory: '@GuzzleHttp\Psr7\HttpFactory' - calls: - - setApiKey: ['%env(LITH_API_KEY)%'] -``` - -## Types Reference - -All types are in the `Lith\Types` namespace: - -- `Provenance` - Audit trail metadata -- `QueryResult` - Query response -- `Collection` - Collection metadata -- `CollectionType` - Document/Edge/Schema enum -- `JournalEntry` - Journal entry -- `JournalOperation` - Operation type enum -- `FunctionalDependency` - Discovered FD -- `ConfidenceLevel` - High/Medium/Low enum -- `NormalForm` - 1NF/2NF/3NF/BCNF enum -- `NormalFormAnalysis` - NF analysis result -- `MigrationStatus` - Migration state -- `MigrationPhase` - Announce/Shadow/Commit/Rollback enum -- `HealthResponse` - Health check response -- `HealthStatus` - Healthy/Degraded/Unhealthy enum - -## License - -PMPL-1.0-or-later diff --git a/lithoglyph/clients/php/composer.json b/lithoglyph/clients/php/composer.json deleted file mode 100644 index 34656c1c..00000000 --- a/lithoglyph/clients/php/composer.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "lith/lith-php", - "description": "PHP client for Lith - the narrative-first, reversible, audit-grade database", - "version": "0.0.6", - "type": "library", - "license": "PMPL-1.0-or-later", - "authors": [ - { - "name": "Jonathan D.A. Jewell", - "email": "jonathan@hyperpolymath.org" - } - ], - "homepage": "https://github.com/hyperpolymath/lith", - "keywords": [ - "database", - "lith", - "gql", - "audit", - "provenance", - "reversible" - ], - "require": { - "php": "^8.1", - "psr/http-client": "^1.0", - "psr/http-factory": "^1.0", - "psr/http-message": "^1.0|^2.0" - }, - "require-dev": { - "guzzlehttp/guzzle": "^7.0", - "phpunit/phpunit": "^10.0", - "phpstan/phpstan": "^1.0" - }, - "autoload": { - "psr-4": { - "Lith\\": "src/" - } - }, - "autoload-dev": { - "psr-4": { - "Lith\\Tests\\": "tests/" - } - }, - "scripts": { - "test": "phpunit", - "analyse": "phpstan analyse src --level=8" - }, - "suggest": { - "guzzlehttp/guzzle": "Required for HTTP requests", - "nyholm/psr7": "Lightweight PSR-7 implementation" - }, - "config": { - "sort-packages": true - }, - "minimum-stability": "stable" -} diff --git a/lithoglyph/clients/php/src/LithClient.php b/lithoglyph/clients/php/src/LithClient.php deleted file mode 100644 index d0863200..00000000 --- a/lithoglyph/clients/php/src/LithClient.php +++ /dev/null @@ -1,387 +0,0 @@ -baseUrl = rtrim($baseUrl, '/'); - $this->httpClient = $httpClient; - $this->requestFactory = $requestFactory; - $this->streamFactory = $streamFactory; - } - - /** - * Create client from environment variables - */ - public static function fromEnv( - ClientInterface $httpClient, - RequestFactoryInterface $requestFactory, - StreamFactoryInterface $streamFactory, - ): self { - $baseUrl = getenv('LITH_URL') ?: 'http://localhost:8080'; - $client = new self($baseUrl, $httpClient, $requestFactory, $streamFactory); - - if ($apiKey = getenv('LITH_API_KEY')) { - $client->setApiKey($apiKey); - } - - return $client; - } - - /** - * Set API key authentication - */ - public function setApiKey(string $apiKey): self - { - $this->apiKey = $apiKey; - $this->bearerToken = null; - return $this; - } - - /** - * Set Bearer token authentication - */ - public function setBearerToken(string $token): self - { - $this->bearerToken = $token; - $this->apiKey = null; - return $this; - } - - // ========================================================================= - // Query Operations - // ========================================================================= - - /** - * Execute an GQL query - */ - public function query(string $gql, ?Provenance $provenance = null, bool $explain = false): QueryResult - { - $body = ['gql' => $gql]; - if ($provenance !== null) { - $body['provenance'] = $provenance->toArray(); - } - if ($explain) { - $body['explain'] = true; - } - - $response = $this->request('POST', '/v1/query', $body); - return QueryResult::fromArray($response); - } - - /** - * Execute a query using the query builder - */ - public function queryWith(QueryBuilder $builder): QueryResult - { - return $this->query($builder->toGql(), $builder->getProvenance()); - } - - /** - * Create a new query builder - */ - public function select(): QueryBuilder - { - return new QueryBuilder(); - } - - /** - * Create a new insert builder - */ - public function insert(): InsertBuilder - { - return new InsertBuilder(); - } - - /** - * Create a new update builder - */ - public function update(): UpdateBuilder - { - return new UpdateBuilder(); - } - - /** - * Create a new delete builder - */ - public function delete(): DeleteBuilder - { - return new DeleteBuilder(); - } - - /** - * Execute an insert using the builder - */ - public function insertWith(InsertBuilder $builder): QueryResult - { - return $this->query($builder->toGql(), $builder->getProvenance()); - } - - /** - * Execute an update using the builder - */ - public function updateWith(UpdateBuilder $builder): QueryResult - { - return $this->query($builder->toGql(), $builder->getProvenance()); - } - - /** - * Execute a delete using the builder - */ - public function deleteWith(DeleteBuilder $builder): QueryResult - { - return $this->query($builder->toGql(), $builder->getProvenance()); - } - - // ========================================================================= - // Collection Operations - // ========================================================================= - - /** - * List all collections - * @return Collection[] - */ - public function listCollections(): array - { - $response = $this->request('GET', '/v1/collections'); - return array_map(fn($item) => Collection::fromArray($item), $response); - } - - /** - * Get a specific collection - */ - public function getCollection(string $name): Collection - { - $response = $this->request('GET', "/v1/collections/{$name}"); - return Collection::fromArray($response); - } - - /** - * Create a new collection - */ - public function createCollection( - string $name, - CollectionType $type = CollectionType::DOCUMENT, - ?array $schema = null - ): Collection { - $body = [ - 'name' => $name, - 'type' => $type->value, - ]; - if ($schema !== null) { - $body['schema'] = $schema; - } - - $response = $this->request('POST', '/v1/collections', $body); - return Collection::fromArray($response); - } - - /** - * Delete a collection - */ - public function deleteCollection(string $name): void - { - $this->request('DELETE', "/v1/collections/{$name}"); - } - - // ========================================================================= - // Journal Operations - // ========================================================================= - - /** - * Get journal entries - * @return JournalEntry[] - */ - public function getJournal(?int $since = null, ?int $limit = null, ?string $collection = null): array - { - $params = []; - if ($since !== null) { - $params['since'] = $since; - } - if ($limit !== null) { - $params['limit'] = $limit; - } - if ($collection !== null) { - $params['collection'] = $collection; - } - - $queryStr = !empty($params) ? '?' . http_build_query($params) : ''; - $response = $this->request('GET', "/v1/journal{$queryStr}"); - - return array_map(fn($item) => JournalEntry::fromArray($item), $response); - } - - // ========================================================================= - // Normalization Operations - // ========================================================================= - - /** - * Discover functional dependencies - * @return FunctionalDependency[] - */ - public function discoverDependencies(string $collection, ?float $confidence = null): array - { - $body = ['collection' => $collection]; - if ($confidence !== null) { - $body['minConfidence'] = $confidence; - } - - $response = $this->request('POST', '/v1/normalize/discover', $body); - return array_map(fn($item) => FunctionalDependency::fromArray($item), $response); - } - - /** - * Analyze normal form - */ - public function analyzeNormalForm(string $collection): NormalFormAnalysis - { - $response = $this->request('POST', '/v1/normalize/analyze', ['collection' => $collection]); - return NormalFormAnalysis::fromArray($response); - } - - // ========================================================================= - // Migration Operations - // ========================================================================= - - /** - * Start a migration - */ - public function startMigration(string $collection, NormalForm $targetForm): MigrationStatus - { - $response = $this->request('POST', '/v1/migrate/start', [ - 'collection' => $collection, - 'targetForm' => $targetForm->value, - ]); - return MigrationStatus::fromArray($response); - } - - /** - * Commit a migration - */ - public function commitMigration(string $migrationId): void - { - $this->request('POST', '/v1/migrate/commit', ['migrationId' => $migrationId]); - } - - // ========================================================================= - // Health Check - // ========================================================================= - - /** - * Check server health - */ - public function health(): HealthResponse - { - $response = $this->request('GET', '/v1/health'); - return HealthResponse::fromArray($response); - } - - // ========================================================================= - // Internal HTTP Methods - // ========================================================================= - - /** - * Make an HTTP request - * @return array - */ - private function request(string $method, string $path, ?array $body = null): array - { - $url = $this->baseUrl . $path; - $request = $this->requestFactory->createRequest($method, $url); - - // Add headers - $request = $request - ->withHeader('Content-Type', 'application/json') - ->withHeader('Accept', 'application/json'); - - if ($this->apiKey !== null) { - $request = $request->withHeader('X-API-Key', $this->apiKey); - } elseif ($this->bearerToken !== null) { - $request = $request->withHeader('Authorization', 'Bearer ' . $this->bearerToken); - } - - // Add body - if ($body !== null) { - $jsonBody = json_encode($body, JSON_THROW_ON_ERROR); - $stream = $this->streamFactory->createStream($jsonBody); - $request = $request->withBody($stream); - } - - // Send request - $response = $this->httpClient->sendRequest($request); - - // Parse response - $statusCode = $response->getStatusCode(); - $responseBody = (string) $response->getBody(); - - if ($statusCode >= 400) { - $errorData = json_decode($responseBody, true) ?: []; - throw new LithException( - $errorData['message'] ?? 'Request failed', - (string) $statusCode, - $errorData['details'] ?? null - ); - } - - if ($responseBody === '') { - return []; - } - - return json_decode($responseBody, true, 512, JSON_THROW_ON_ERROR); - } -} diff --git a/lithoglyph/clients/php/src/Query/QueryBuilder.php b/lithoglyph/clients/php/src/Query/QueryBuilder.php deleted file mode 100644 index 8a23eb35..00000000 --- a/lithoglyph/clients/php/src/Query/QueryBuilder.php +++ /dev/null @@ -1,496 +0,0 @@ -'; - case GE = '>='; - case LIKE = 'LIKE'; - case IN = 'IN'; -} - -/** - * Filter expression interface - */ -interface FilterExpr -{ - public function toGql(): string; -} - -/** - * Field comparison filter - */ -final class FieldFilter implements FilterExpr -{ - public function __construct( - public readonly string $field, - public readonly CompareOp $op, - public readonly mixed $value, - ) {} - - public function toGql(): string - { - $valueStr = match (true) { - is_string($this->value) => '"' . addslashes($this->value) . '"', - is_bool($this->value) => $this->value ? 'true' : 'false', - is_null($this->value) => 'null', - is_array($this->value) => '(' . implode(', ', array_map( - fn($v) => is_string($v) ? '"' . addslashes($v) . '"' : (string) $v, - $this->value - )) . ')', - default => (string) $this->value, - }; - - return "{$this->field} {$this->op->value} {$valueStr}"; - } -} - -/** - * AND filter - */ -final class AndFilter implements FilterExpr -{ - public function __construct( - public readonly FilterExpr $left, - public readonly FilterExpr $right, - ) {} - - public function toGql(): string - { - return '(' . $this->left->toGql() . ' AND ' . $this->right->toGql() . ')'; - } -} - -/** - * OR filter - */ -final class OrFilter implements FilterExpr -{ - public function __construct( - public readonly FilterExpr $left, - public readonly FilterExpr $right, - ) {} - - public function toGql(): string - { - return '(' . $this->left->toGql() . ' OR ' . $this->right->toGql() . ')'; - } -} - -/** - * NOT filter - */ -final class NotFilter implements FilterExpr -{ - public function __construct( - public readonly FilterExpr $filter, - ) {} - - public function toGql(): string - { - return 'NOT (' . $this->filter->toGql() . ')'; - } -} - -/** - * Fluent query builder for SELECT queries - */ -final class QueryBuilder -{ - private ?string $collection = null; - /** @var string[]|null */ - private ?array $fields = null; - private ?FilterExpr $filter = null; - private ?int $limit = null; - private ?int $offset = null; - private ?string $orderByField = null; - private bool $orderAscending = true; - private ?Provenance $provenance = null; - - /** - * Set the collection to query - */ - public function from(string $collection): self - { - $this->collection = $collection; - return $this; - } - - /** - * Alias for from() - */ - public function collection(string $collection): self - { - return $this->from($collection); - } - - /** - * Set fields to select - * @param string[] $fields - */ - public function select(array $fields): self - { - $this->fields = $fields; - return $this; - } - - /** - * Add a WHERE filter expression - */ - public function where(FilterExpr $filter): self - { - if ($this->filter === null) { - $this->filter = $filter; - } else { - $this->filter = new AndFilter($this->filter, $filter); - } - return $this; - } - - /** - * Add a field comparison filter (convenience method) - */ - public function whereField(string $field, CompareOp|string $op, mixed $value): self - { - $op = is_string($op) ? CompareOp::from($op) : $op; - return $this->where(new FieldFilter($field, $op, $value)); - } - - /** - * Set limit - */ - public function limit(int $limit): self - { - $this->limit = $limit; - return $this; - } - - /** - * Set offset - */ - public function offset(int $offset): self - { - $this->offset = $offset; - return $this; - } - - /** - * Set order by - */ - public function orderBy(string $field, bool $ascending = true): self - { - $this->orderByField = $field; - $this->orderAscending = $ascending; - return $this; - } - - /** - * Add provenance metadata - */ - public function withProvenance(string|Provenance $actorOrProvenance, ?string $rationale = null): self - { - if ($actorOrProvenance instanceof Provenance) { - $this->provenance = $actorOrProvenance; - } else { - $this->provenance = new Provenance($actorOrProvenance, $rationale ?? ''); - } - return $this; - } - - /** - * Build the GQL query string - */ - public function toGql(): string - { - if ($this->collection === null) { - throw new \RuntimeException('Collection is required'); - } - - $fieldsStr = $this->fields !== null ? implode(', ', $this->fields) : '*'; - $query = "SELECT {$fieldsStr} FROM {$this->collection}"; - - if ($this->filter !== null) { - $query .= ' WHERE ' . $this->filter->toGql(); - } - - if ($this->orderByField !== null) { - $dir = $this->orderAscending ? 'ASC' : 'DESC'; - $query .= " ORDER BY {$this->orderByField} {$dir}"; - } - - if ($this->limit !== null) { - $query .= " LIMIT {$this->limit}"; - } - - if ($this->offset !== null) { - $query .= " OFFSET {$this->offset}"; - } - - if ($this->provenance !== null) { - $query .= ' WITH PROVENANCE { actor: "' . addslashes($this->provenance->actor) . '", rationale: "' . addslashes($this->provenance->rationale) . '" }'; - } - - return $query; - } - - /** - * Get the provenance (for client to use) - */ - public function getProvenance(): ?Provenance - { - return $this->provenance; - } -} - -/** - * Fluent builder for INSERT statements - */ -final class InsertBuilder -{ - private ?string $collection = null; - /** @var array|null */ - private ?array $document = null; - private ?Provenance $provenance = null; - - /** - * Set the collection to insert into - */ - public function into(string $collection): self - { - $this->collection = $collection; - return $this; - } - - /** - * Set the document to insert - * @param array $document - */ - public function values(array $document): self - { - $this->document = $document; - return $this; - } - - /** - * Add provenance metadata - */ - public function withProvenance(string|Provenance $actorOrProvenance, ?string $rationale = null): self - { - if ($actorOrProvenance instanceof Provenance) { - $this->provenance = $actorOrProvenance; - } else { - $this->provenance = new Provenance($actorOrProvenance, $rationale ?? ''); - } - return $this; - } - - /** - * Build the GQL insert string - */ - public function toGql(): string - { - if ($this->collection === null) { - throw new \RuntimeException('Collection is required'); - } - if ($this->document === null) { - throw new \RuntimeException('Document is required'); - } - - $docJson = json_encode($this->document, JSON_THROW_ON_ERROR); - $query = "INSERT INTO {$this->collection} {$docJson}"; - - if ($this->provenance !== null) { - $query .= ' WITH PROVENANCE { actor: "' . addslashes($this->provenance->actor) . '", rationale: "' . addslashes($this->provenance->rationale) . '" }'; - } - - return $query; - } - - public function getProvenance(): ?Provenance - { - return $this->provenance; - } -} - -/** - * Fluent builder for UPDATE statements - */ -final class UpdateBuilder -{ - private ?string $collection = null; - /** @var array */ - private array $sets = []; - private ?FilterExpr $filter = null; - private ?Provenance $provenance = null; - - /** - * Set the collection to update - */ - public function collection(string $collection): self - { - $this->collection = $collection; - return $this; - } - - /** - * Add a SET clause - */ - public function set(string $field, mixed $value): self - { - $this->sets[$field] = $value; - return $this; - } - - /** - * Add a WHERE filter - */ - public function where(FilterExpr $filter): self - { - if ($this->filter === null) { - $this->filter = $filter; - } else { - $this->filter = new AndFilter($this->filter, $filter); - } - return $this; - } - - /** - * Add provenance metadata - */ - public function withProvenance(string|Provenance $actorOrProvenance, ?string $rationale = null): self - { - if ($actorOrProvenance instanceof Provenance) { - $this->provenance = $actorOrProvenance; - } else { - $this->provenance = new Provenance($actorOrProvenance, $rationale ?? ''); - } - return $this; - } - - /** - * Build the GQL update string - */ - public function toGql(): string - { - if ($this->collection === null) { - throw new \RuntimeException('Collection is required'); - } - if (empty($this->sets)) { - throw new \RuntimeException('At least one SET clause is required'); - } - - $setClauses = []; - foreach ($this->sets as $field => $value) { - $valueStr = match (true) { - is_string($value) => '"' . addslashes($value) . '"', - is_bool($value) => $value ? 'true' : 'false', - is_null($value) => 'null', - default => (string) $value, - }; - $setClauses[] = "{$field} = {$valueStr}"; - } - - $query = "UPDATE {$this->collection} SET " . implode(', ', $setClauses); - - if ($this->filter !== null) { - $query .= ' WHERE ' . $this->filter->toGql(); - } - - if ($this->provenance !== null) { - $query .= ' WITH PROVENANCE { actor: "' . addslashes($this->provenance->actor) . '", rationale: "' . addslashes($this->provenance->rationale) . '" }'; - } - - return $query; - } - - public function getProvenance(): ?Provenance - { - return $this->provenance; - } -} - -/** - * Fluent builder for DELETE statements - */ -final class DeleteBuilder -{ - private ?string $collection = null; - private ?FilterExpr $filter = null; - private ?Provenance $provenance = null; - - /** - * Set the collection to delete from - */ - public function from(string $collection): self - { - $this->collection = $collection; - return $this; - } - - /** - * Add a WHERE filter - */ - public function where(FilterExpr $filter): self - { - if ($this->filter === null) { - $this->filter = $filter; - } else { - $this->filter = new AndFilter($this->filter, $filter); - } - return $this; - } - - /** - * Add provenance metadata - */ - public function withProvenance(string|Provenance $actorOrProvenance, ?string $rationale = null): self - { - if ($actorOrProvenance instanceof Provenance) { - $this->provenance = $actorOrProvenance; - } else { - $this->provenance = new Provenance($actorOrProvenance, $rationale ?? ''); - } - return $this; - } - - /** - * Build the GQL delete string - */ - public function toGql(): string - { - if ($this->collection === null) { - throw new \RuntimeException('Collection is required'); - } - - $query = "DELETE FROM {$this->collection}"; - - if ($this->filter !== null) { - $query .= ' WHERE ' . $this->filter->toGql(); - } - - if ($this->provenance !== null) { - $query .= ' WITH PROVENANCE { actor: "' . addslashes($this->provenance->actor) . '", rationale: "' . addslashes($this->provenance->rationale) . '" }'; - } - - return $query; - } - - public function getProvenance(): ?Provenance - { - return $this->provenance; - } -} diff --git a/lithoglyph/clients/php/src/Types/Collection.php b/lithoglyph/clients/php/src/Types/Collection.php deleted file mode 100644 index 63514ada..00000000 --- a/lithoglyph/clients/php/src/Types/Collection.php +++ /dev/null @@ -1,40 +0,0 @@ - $this->actor, - 'rationale' => $this->rationale, - 'timestamp' => $this->timestamp, - 'source' => $this->source, - ], fn($v) => $v !== null); - } - - public function jsonSerialize(): array - { - return $this->toArray(); - } -} diff --git a/lithoglyph/clients/php/src/Types/QueryResult.php b/lithoglyph/clients/php/src/Types/QueryResult.php deleted file mode 100644 index 375237f0..00000000 --- a/lithoglyph/clients/php/src/Types/QueryResult.php +++ /dev/null @@ -1,33 +0,0 @@ -> $rows - */ - public function __construct( - public readonly array $rows, - public readonly int $rowCount, - public readonly int $journalSeq, - public readonly ?Provenance $provenance = null, - ) {} - - public static function fromArray(array $data): self - { - return new self( - rows: $data['rows'] ?? [], - rowCount: (int) ($data['rowCount'] ?? 0), - journalSeq: (int) ($data['journalSeq'] ?? 0), - provenance: isset($data['provenance']) ? Provenance::fromArray($data['provenance']) : null, - ); - } -} diff --git a/lithoglyph/clients/rescript/README.md b/lithoglyph/clients/rescript/README.md deleted file mode 100644 index b3a79bae..00000000 --- a/lithoglyph/clients/rescript/README.md +++ /dev/null @@ -1,279 +0,0 @@ -# @lithoglyph/rescript - -Type-safe ReScript client for the Lithoglyph multi-protocol API server. - -Supports REST, GraphQL, and WebSocket (subscriptions) protocols. - -SPDX-License-Identifier: PMPL-1.0-or-later - -## Installation - -### With Deno (Recommended) - -```bash -# Add to deno.json imports -{ - "imports": { - "@lithoglyph/rescript": "jsr:@lithoglyph/rescript@1.0.0" - } -} -``` - -### With ReScript + Deno - -Add to `rescript.json`: - -```json -{ - "bs-dependencies": ["@lithoglyph/rescript", "@rescript/core"] -} -``` - -## Quick Start - -```rescript -open Lithoglyph -open Lithoglyph_Types -open Lithoglyph_Query - -// Create client -let client = Lithoglyph.make(~baseUrl="http://localhost:8080") - -// Or from environment (reads LITHOGLYPH_URL / LITHOGLYPH_API_KEY) -let client = Lithoglyph.fromEnv() - -// Check health -let healthResult = await client->Lithoglyph.health -switch healthResult { -| Ok(h) => Console.log(`Server: ${h.status == Healthy ? "healthy" : "unhealthy"}`) -| Error(e) => Console.error(e.message) -} -``` - -## Query Examples - -### Using Query Builder - -```rescript -let query = Lithoglyph_Query.make() - ->from("articles") - ->select(["id", "title", "author"]) - ->whereField("status", Eq, JSON.Encode.string("published")) - ->whereField("views", Gt, JSON.Encode.int(100)) - ->orderBy("createdAt", ~ascending=false) - ->limit(10) - ->withProvenance({ - actor: "editor@news.org", - rationale: "Daily review of popular articles" - }) - -let result = await client->Lithoglyph.queryWith(query) -``` - -### Raw GQL - -```rescript -let result = await client->Lithoglyph.query( - ~gql=`SELECT * FROM articles WHERE status = "published" LIMIT 10`, - ~provenance={ - actor: "editor@news.org", - rationale: "Daily review" - } -) - -switch result { -| Ok(r) => Console.log(`Found ${Int.toString(r.rowCount)} articles`) -| Error(e) => Console.error(e.message) -} -``` - -### Via GraphQL - -```rescript -let result = await client->Lithoglyph.queryGraphQL( - ~gql=`SELECT * FROM articles LIMIT 5` -) -``` - -## EXPLAIN - -```rescript -let plan = await client->Lithoglyph.explain( - ~gql=`SELECT * FROM articles WHERE status = "published"`, - ~analyze=true -) - -switch plan { -| Ok(e) => { - Console.log(`Estimated cost: ${Float.toString(e.plan.estimatedCost)}`) - e.plan.steps->Array.forEach(step => - Console.log(` Step: ${step.collection->Option.getOr("?")}`) - ) - } -| Error(e) => Console.error(e.message) -} -``` - -## Collection Operations - -```rescript -// List collections -let collections = await client->Lithoglyph.listCollections - -// Create collection -let newCol = await client->Lithoglyph.createCollection( - ~name="users", - ~collectionType=Document, -) - -// Get collection -let col = await client->Lithoglyph.getCollection(~name="articles") - -// Delete collection -let _ = await client->Lithoglyph.deleteCollection(~name="temp_data") -``` - -## Journal Operations - -```rescript -let journal = await client->Lithoglyph.getJournal( - ~since=1000, - ~limit=50, - ~collection="articles" -) - -switch journal { -| Ok(j) => { - j.entries->Array.forEach(entry => - Console.log(`[${Int.toString(entry.seq)}] ${entry.collection->Option.getOr("?")}`) - ) - if j.hasMore { Console.log("More entries available...") } - } -| Error(e) => Console.error(e.message) -} -``` - -## Normalization - -```rescript -// Discover functional dependencies -let fds = await client->Lithoglyph.discoverDependencies( - ~collection="orders", - ~confidence=0.95 -) - -// Analyze normal form -let analysis = await client->Lithoglyph.analyzeNormalForm(~collection="orders") -``` - -## Migrations (Announce-Shadow-Commit) - -```rescript -// Start migration -let migration = await client->Lithoglyph.startMigration( - ~collection="orders", - ~targetForm=BCNF -) - -switch migration { -| Ok(m) => { - Console.log(`Migration ${m.id}: ${m.narrative}`) - - // Advance to shadow (dual-write) phase - let _ = await client->Lithoglyph.advanceToShadow(~migrationId=m.id) - - // When ready, commit - let _ = await client->Lithoglyph.commitMigration(~migrationId=m.id) - - // Or abort if something goes wrong - // let _ = await client->Lithoglyph.abortMigration(~migrationId=m.id) - } -| Error(e) => Console.error(e.message) -} -``` - -## WebSocket Subscriptions - -### Journal Streaming - -```rescript -open Lithoglyph_Subscriptions - -let handle = subscribeJournal( - ~baseUrl="http://localhost:8080", - ~collection="articles", - ~onEntry=entry => { - Console.log(`[${Int.toString(entry.seq)}] New journal entry`) - }, - ~onError=err => Console.error(err), -) - -// Later: stop streaming -handle.unsubscribe() -``` - -### Migration Progress - -```rescript -open Lithoglyph_Subscriptions - -let handle = subscribeMigrationProgress( - ~baseUrl="http://localhost:8080", - ~migrationId="mig-001", - ~onProgress=progress => { - Console.log(`Migration ${Float.toString(progress.progress * 100.0)}%: ${progress.message}`) - }, -) -``` - -## Authentication - -```rescript -// API Key -let client = Lithoglyph.make( - ~baseUrl="http://localhost:8080", - ~auth=ApiKey("your-api-key") -) - -// Bearer Token (JWT) -let client = Lithoglyph.make( - ~baseUrl="http://localhost:8080", - ~auth=Bearer("your-jwt-token") -) -``` - -## Module Structure - -| Module | Purpose | -|--------|---------| -| `Lithoglyph` | Main client (REST + GraphQL) | -| `Lithoglyph_Types` | All type definitions | -| `Lithoglyph_Query` | GQL query builder | -| `Lithoglyph_Subscriptions` | WebSocket real-time subscriptions | - -## Error Handling - -All API methods return `Result.t<'a, apiError>`: - -```rescript -switch result { -| Ok(data) => // Handle success -| Error({code, message, details}) => { - Console.error(`Error ${code}: ${message}`) - } -} -``` - -## Backwards Compatibility - -The client reads `LITHOGLYPH_URL` / `LITHOGLYPH_API_KEY` environment variables, -falling back to the legacy `LITH_URL` / `LITH_API_KEY` names. - -The old `Lith` / `Lith_Types` / `Lith_Query` modules are still present -for backwards compatibility but should be considered deprecated. - -## License - -PMPL-1.0-or-later (Palimpsest License) - -Copyright (c) 2026 Jonathan D.A. Jewell diff --git a/lithoglyph/clients/rescript/deno.json b/lithoglyph/clients/rescript/deno.json deleted file mode 100644 index 430e8b33..00000000 --- a/lithoglyph/clients/rescript/deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@lithoglyph/rescript", - "version": "1.0.0", - "exports": "./src/Lithoglyph.res.js", - "tasks": { - "build": "rescript build", - "clean": "rescript clean", - "dev": "rescript build -w", - "test": "deno test --allow-net tests/" - }, - "imports": { - "@rescript/core": "npm:@rescript/core@^1.0.0" - }, - "compilerOptions": { - "lib": [ - "deno.window", - "dom" - ] - } -} diff --git a/lithoglyph/clients/rescript/rescript.json b/lithoglyph/clients/rescript/rescript.json deleted file mode 100644 index b2595dde..00000000 --- a/lithoglyph/clients/rescript/rescript.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "@lithoglyph/rescript", - "version": "1.0.0", - "sources": [ - { - "dir": "src", - "subdirs": true - } - ], - "package-specs": [ - { - "module": "es6", - "in-source": true - } - ], - "suffix": ".res.js", - "bs-dependencies": [ - "@rescript/core" - ], - "bsc-flags": [ - "-open RescriptCore" - ] -} diff --git a/lithoglyph/clients/rescript/src/Lith.res b/lithoglyph/clients/rescript/src/Lith.res deleted file mode 100644 index eb15e396..00000000 --- a/lithoglyph/clients/rescript/src/Lith.res +++ /dev/null @@ -1,496 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Lith ReScript Client -// -// Type-safe client for Lith REST API -// Compatible with Deno runtime - -open Lith_Types -open Lith_Query - -// ============================================================================= -// HTTP Helpers -// ============================================================================= - -@val external fetch: (string, 'options) => promise<'response> = "fetch" - -type fetchOptions = { - method: string, - headers: Dict.t, - body?: string, -} - -type fetchResponse = { - ok: bool, - status: int, - json: unit => promise, - text: unit => promise, -} - -// ============================================================================= -// Client -// ============================================================================= - -type t = { - config: config, -} - -/** Create a new Lith client */ -let make = (~baseUrl, ~auth=?, ~timeout=?, ~retries=?) => { - config: { - baseUrl, - auth, - timeout, - retries, - }, -} - -/** Create client from environment (Deno) */ -let fromEnv = () => { - // Access Deno.env in Deno runtime - let baseUrl = %raw(`Deno.env.get("LITH_URL") || "http://localhost:8080"`) - let apiKey = %raw(`Deno.env.get("LITH_API_KEY")`) - - make( - ~baseUrl, - ~auth=?switch apiKey { - | Some(key) => Some(ApiKey(key)) - | None => None - }, - ) -} - -// Internal: Build headers for requests -let buildHeaders = client => { - let headers = Dict.make() - headers->Dict.set("Content-Type", "application/json") - headers->Dict.set("Accept", "application/json") - - switch client.config.auth { - | Some(ApiKey(key)) => headers->Dict.set("X-API-Key", key) - | Some(Bearer(token)) => headers->Dict.set("Authorization", `Bearer ${token}`) - | Some(NoAuth) | None => () - } - - headers -} - -// Internal: Make HTTP request -let request = async (client, ~method, ~path, ~body=?) => { - let url = `${client.config.baseUrl}${path}` - let headers = buildHeaders(client) - - let options: fetchOptions = { - method, - headers, - body: ?body->Option.map(JSON.stringify), - } - - let response: fetchResponse = await fetch(url, options) - - if !response.ok { - let text = await response.text() - Error({ - code: Int.toString(response.status), - message: text, - details: None, - }) - } else { - let json = await response.json() - Ok(json) - } -} - -// ============================================================================= -// Query Operations -// ============================================================================= - -/** Execute an GQL query */ -let query = async (client, ~gql, ~provenance=?, ~explain=?) => { - let body: JSON.t = JSON.Encode.object([ - ("gql", JSON.Encode.string(gql)), - ...switch provenance { - | Some(p) => [ - ( - "provenance", - JSON.Encode.object([ - ("actor", JSON.Encode.string(p.actor)), - ("rationale", JSON.Encode.string(p.rationale)), - ]), - ), - ] - | None => [] - }, - ...switch explain { - | Some(true) => [("explain", JSON.Encode.bool(true))] - | Some(false) | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/query", ~body) - - result->Result.map(json => { - // Parse query result - let obj = json->JSON.Decode.object->Option.getExn - { - rows: obj - ->Dict.get("rows") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(row => row->JSON.Decode.object->Option.getOr(Dict.make())), - rowCount: obj - ->Dict.get("rowCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - journalSeq: obj - ->Dict.get("journalSeq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - provenance: None, - } - }) -} - -/** Execute a query using the query builder */ -let queryWith = async (client, builder) => { - let gql = builder->toGql - let provenance = builder.provenance - await query(client, ~gql, ~provenance?) -} - -// ============================================================================= -// Collection Operations -// ============================================================================= - -/** List all collections */ -let listCollections = async client => { - let result = await request(client, ~method="GET", ~path="/v1/collections") - - result->Result.map(json => { - json - ->JSON.Decode.array - ->Option.getOr([]) - ->Array.map(item => { - let obj = item->JSON.Decode.object->Option.getExn - { - name: obj->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch obj - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" => Edge - | "SCHEMA" => Schema - | _ => Document - }, - documentCount: obj - ->Dict.get("documentCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - schema: obj->Dict.get("schema"), - } - }) - }) -} - -/** Get a specific collection */ -let getCollection = async (client, ~name) => { - let result = await request(client, ~method="GET", ~path=`/v1/collections/${name}`) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - name: obj->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch obj - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" => Edge - | "SCHEMA" => Schema - | _ => Document - }, - documentCount: obj - ->Dict.get("documentCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - schema: obj->Dict.get("schema"), - } - }) -} - -/** Create a new collection */ -let createCollection = async (client, ~name, ~collectionType=Document, ~schema=?) => { - let typeStr = switch collectionType { - | Document => "DOCUMENT" - | Edge => "EDGE" - | Schema => "SCHEMA" - } - - let body: JSON.t = JSON.Encode.object([ - ("name", JSON.Encode.string(name)), - ("type", JSON.Encode.string(typeStr)), - ...switch schema { - | Some(s) => [("schema", s)] - | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/collections", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - name: obj->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch obj - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" => Edge - | "SCHEMA" => Schema - | _ => Document - }, - documentCount: 0, - schema: obj->Dict.get("schema"), - } - }) -} - -/** Delete a collection */ -let deleteCollection = async (client, ~name) => { - let result = await request(client, ~method="DELETE", ~path=`/v1/collections/${name}`) - result->Result.map(_ => ()) -} - -// ============================================================================= -// Journal Operations -// ============================================================================= - -/** Get journal entries */ -let getJournal = async (client, ~since=?, ~limit=?, ~collection=?) => { - let params = [] - switch since { - | Some(seq) => params->Array.push(`since=${Int.toString(seq)}`)->ignore - | None => () - } - switch limit { - | Some(n) => params->Array.push(`limit=${Int.toString(n)}`)->ignore - | None => () - } - switch collection { - | Some(c) => params->Array.push(`collection=${c}`)->ignore - | None => () - } - - let queryStr = params->Array.length > 0 ? `?${params->Array.join("&")}` : "" - - let result = await request(client, ~method="GET", ~path=`/v1/journal${queryStr}`) - - result->Result.map(json => { - json - ->JSON.Decode.array - ->Option.getOr([]) - ->Array.map(item => { - let obj = item->JSON.Decode.object->Option.getExn - { - seq: obj - ->Dict.get("seq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - timestamp: obj->Dict.get("timestamp")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - operation: switch obj - ->Dict.get("operation") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("INSERT") { - | "UPDATE" => Update - | "DELETE" => Delete - | "CREATE_COLLECTION" => CreateCollection - | "DROP_COLLECTION" => DropCollection - | "MIGRATION_START" => MigrationStart - | "MIGRATION_COMMIT" => MigrationCommit - | _ => Insert - }, - collection: obj->Dict.get("collection")->Option.flatMap(JSON.Decode.string), - documentId: obj->Dict.get("documentId")->Option.flatMap(JSON.Decode.string), - provenance: None, - } - }) - }) -} - -// ============================================================================= -// Normalization Operations -// ============================================================================= - -/** Discover functional dependencies in a collection */ -let discoverDependencies = async (client, ~collection, ~confidence=?) => { - let body: JSON.t = JSON.Encode.object([ - ("collection", JSON.Encode.string(collection)), - ...switch confidence { - | Some(c) => [("minConfidence", JSON.Encode.float(c))] - | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/normalize/discover", ~body) - - result->Result.map(json => { - json - ->JSON.Decode.array - ->Option.getOr([]) - ->Array.map(item => { - let obj = item->JSON.Decode.object->Option.getExn - { - determinant: obj - ->Dict.get("determinant") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - dependent: obj->Dict.get("dependent")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - confidence: switch obj - ->Dict.get("confidence") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("MEDIUM") { - | "HIGH" => High - | "LOW" => Low - | _ => Medium - }, - sampleSize: obj - ->Dict.get("sampleSize") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - } - }) - }) -} - -/** Analyze normal form of a collection */ -let analyzeNormalForm = async (client, ~collection) => { - let body: JSON.t = JSON.Encode.object([("collection", JSON.Encode.string(collection))]) - - let result = await request(client, ~method="POST", ~path="/v1/normalize/analyze", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - currentForm: switch obj - ->Dict.get("currentForm") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("1NF") { - | "2NF" => NF2 - | "3NF" => NF3 - | "BCNF" => BCNF - | _ => NF1 - }, - targetForm: switch obj - ->Dict.get("targetForm") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("BCNF") { - | "1NF" => NF1 - | "2NF" => NF2 - | "3NF" => NF3 - | _ => BCNF - }, - violations: obj - ->Dict.get("violations") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - recommendations: obj - ->Dict.get("recommendations") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - } - }) -} - -// ============================================================================= -// Migration Operations -// ============================================================================= - -/** Start a migration */ -let startMigration = async (client, ~collection, ~targetForm) => { - let targetStr = switch targetForm { - | NF1 => "1NF" - | NF2 => "2NF" - | NF3 => "3NF" - | BCNF => "BCNF" - } - - let body: JSON.t = JSON.Encode.object([ - ("collection", JSON.Encode.string(collection)), - ("targetForm", JSON.Encode.string(targetStr)), - ]) - - let result = await request(client, ~method="POST", ~path="/v1/migrate/start", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - id: obj->Dict.get("id")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - phase: switch obj - ->Dict.get("phase") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("ANNOUNCE") { - | "SHADOW" => Shadow - | "COMMIT" => Commit - | "ROLLBACK" => Rollback - | _ => Announce - }, - collection: obj - ->Dict.get("collection") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - startedAt: obj->Dict.get("startedAt")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - narrative: obj->Dict.get("narrative")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - } - }) -} - -/** Commit a migration */ -let commitMigration = async (client, ~migrationId) => { - let body: JSON.t = JSON.Encode.object([("migrationId", JSON.Encode.string(migrationId))]) - - let result = await request(client, ~method="POST", ~path="/v1/migrate/commit", ~body) - result->Result.map(_ => ()) -} - -// ============================================================================= -// Health Check -// ============================================================================= - -/** Check server health */ -let health = async client => { - let result = await request(client, ~method="GET", ~path="/v1/health") - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - status: switch obj - ->Dict.get("status") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("UNHEALTHY") { - | "HEALTHY" => Healthy - | "DEGRADED" => Degraded - | _ => Unhealthy - }, - version: obj->Dict.get("version")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - uptimeSeconds: obj - ->Dict.get("uptimeSeconds") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - } - }) -} - -// ============================================================================= -// Re-exports -// ============================================================================= - -module Types = Lith_Types -module Query = Lith_Query diff --git a/lithoglyph/clients/rescript/src/Lith_Query.res b/lithoglyph/clients/rescript/src/Lith_Query.res deleted file mode 100644 index 63931f01..00000000 --- a/lithoglyph/clients/rescript/src/Lith_Query.res +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Lith ReScript Client - Query Builder - -open Lith_Types - -// ============================================================================= -// Comparison Operators -// ============================================================================= - -type compareOp = - | Eq - | Ne - | Lt - | Le - | Gt - | Ge - | Like - | In - -let compareOpToString = op => - switch op { - | Eq => "=" - | Ne => "!=" - | Lt => "<" - | Le => "<=" - | Gt => ">" - | Ge => ">=" - | Like => "LIKE" - | In => "IN" - } - -// ============================================================================= -// Filter Expressions -// ============================================================================= - -type rec filterExpr = - | Field(string, compareOp, JSON.t) - | And(filterExpr, filterExpr) - | Or(filterExpr, filterExpr) - | Not(filterExpr) - -let rec filterToGql = filter => - switch filter { - | Field(name, op, value) => { - let valueStr = switch value { - | JSON.String(s) => `"${s}"` - | JSON.Number(n) => Float.toString(n) - | JSON.Boolean(b) => b ? "true" : "false" - | JSON.Null => "null" - | _ => JSON.stringify(value) - } - `${name} ${compareOpToString(op)} ${valueStr}` - } - | And(a, b) => `(${filterToGql(a)} AND ${filterToGql(b)})` - | Or(a, b) => `(${filterToGql(a)} OR ${filterToGql(b)})` - | Not(f) => `NOT (${filterToGql(f)})` - } - -// ============================================================================= -// Query Builder -// ============================================================================= - -type queryBuilder = { - mutable collection: option, - mutable fields: option>, - mutable filter: option, - mutable limit: option, - mutable offset: option, - mutable orderBy: option<(string, bool)>, // (field, ascending) - mutable provenance: option, -} - -/** Create a new query builder */ -let make = () => { - collection: None, - fields: None, - filter: None, - limit: None, - offset: None, - orderBy: None, - provenance: None, -} - -/** Set the collection to query */ -let from = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -/** Set fields to select (default: all) */ -let select = (builder, fieldList) => { - builder.fields = Some(fieldList) - builder -} - -/** Add a WHERE filter */ -let where = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -/** Add a field comparison filter (convenience) */ -let whereField = (builder, fieldName, op, value) => { - let newFilter = Field(fieldName, op, value) - builder.filter = switch builder.filter { - | Some(existing) => Some(And(existing, newFilter)) - | None => Some(newFilter) - } - builder -} - -/** Set limit */ -let limit = (builder, n) => { - builder.limit = Some(n) - builder -} - -/** Set offset */ -let offset = (builder, n) => { - builder.offset = Some(n) - builder -} - -/** Set order by */ -let orderBy = (builder, field, ~ascending=true) => { - builder.orderBy = Some((field, ascending)) - builder -} - -/** Add provenance metadata */ -let withProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -/** Build the GQL query string */ -let toGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let fieldsStr = switch builder.fields { - | Some(fields) => fields->Array.join(", ") - | None => "*" - } - - let mut query = `SELECT ${fieldsStr} FROM ${collection}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.orderBy { - | Some((field, asc)) => { - let dir = asc ? "ASC" : "DESC" - query = query ++ ` ORDER BY ${field} ${dir}` - } - | None => () - } - - switch builder.limit { - | Some(n) => query = query ++ ` LIMIT ${Int.toString(n)}` - | None => () - } - - switch builder.offset { - | Some(n) => query = query ++ ` OFFSET ${Int.toString(n)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// Insert Builder -// ============================================================================= - -type insertBuilder = { - mutable collection: option, - mutable document: option, - mutable provenance: option, -} - -let makeInsert = () => { - collection: None, - document: None, - provenance: None, -} - -let into = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -let values = (builder, doc) => { - builder.document = Some(doc) - builder -} - -let insertWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -let insertToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let doc = switch builder.document { - | Some(d) => JSON.stringify(d) - | None => panic("Document is required") - } - - let mut query = `INSERT INTO ${collection} ${doc}` - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// Update Builder -// ============================================================================= - -type updateBuilder = { - mutable collection: option, - mutable sets: array<(string, JSON.t)>, - mutable filter: option, - mutable provenance: option, -} - -let makeUpdate = () => { - collection: None, - sets: [], - filter: None, - provenance: None, -} - -let updateCollection = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -let set = (builder, field, value) => { - builder.sets = builder.sets->Array.concat([(field, value)]) - builder -} - -let updateWhere = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -let updateWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -let updateToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - if builder.sets->Array.length == 0 { - panic("At least one SET clause is required") - } - - let setsClauses = - builder.sets - ->Array.map(((field, value)) => { - let valueStr = switch value { - | JSON.String(s) => `"${s}"` - | JSON.Number(n) => Float.toString(n) - | JSON.Boolean(b) => b ? "true" : "false" - | JSON.Null => "null" - | _ => JSON.stringify(value) - } - `${field} = ${valueStr}` - }) - ->Array.join(", ") - - let mut query = `UPDATE ${collection} SET ${setsClauses}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// Delete Builder -// ============================================================================= - -type deleteBuilder = { - mutable collection: option, - mutable filter: option, - mutable provenance: option, -} - -let makeDelete = () => { - collection: None, - filter: None, - provenance: None, -} - -let deleteFrom = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -let deleteWhere = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -let deleteWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -let deleteToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let mut query = `DELETE FROM ${collection}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} diff --git a/lithoglyph/clients/rescript/src/Lith_Types.res b/lithoglyph/clients/rescript/src/Lith_Types.res deleted file mode 100644 index f0000729..00000000 --- a/lithoglyph/clients/rescript/src/Lith_Types.res +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Lith ReScript Client - Type Definitions - -// ============================================================================= -// Core Types -// ============================================================================= - -/** Provenance metadata for audit trail */ -type provenance = { - actor: string, - rationale: string, - timestamp?: string, - source?: string, -} - -/** Query result row */ -type row = Dict.t - -/** Query result */ -type queryResult = { - rows: array, - rowCount: int, - journalSeq: int, - provenance?: provenance, -} - -/** Collection type */ -type collectionType = - | @as("DOCUMENT") Document - | @as("EDGE") Edge - | @as("SCHEMA") Schema - -/** Collection metadata */ -type collection = { - name: string, - @as("type") collectionType: collectionType, - documentCount: int, - schema?: JSON.t, -} - -/** Journal operation type */ -type journalOperation = - | @as("INSERT") Insert - | @as("UPDATE") Update - | @as("DELETE") Delete - | @as("CREATE_COLLECTION") CreateCollection - | @as("DROP_COLLECTION") DropCollection - | @as("MIGRATION_START") MigrationStart - | @as("MIGRATION_COMMIT") MigrationCommit - -/** Journal entry */ -type journalEntry = { - seq: int, - timestamp: string, - operation: journalOperation, - collection?: string, - documentId?: string, - provenance?: provenance, -} - -// ============================================================================= -// Normalization Types -// ============================================================================= - -/** Normal form level */ -type normalForm = - | @as("1NF") NF1 - | @as("2NF") NF2 - | @as("3NF") NF3 - | @as("BCNF") BCNF - -/** Confidence level for discovered dependencies */ -type confidenceLevel = - | @as("HIGH") High - | @as("MEDIUM") Medium - | @as("LOW") Low - -/** Functional dependency */ -type functionalDependency = { - determinant: array, - dependent: string, - confidence: confidenceLevel, - sampleSize: int, -} - -/** Normal form analysis result */ -type normalFormAnalysis = { - currentForm: normalForm, - targetForm: normalForm, - violations: array, - recommendations: array, -} - -// ============================================================================= -// Migration Types -// ============================================================================= - -/** Migration phase */ -type migrationPhase = - | @as("ANNOUNCE") Announce - | @as("SHADOW") Shadow - | @as("COMMIT") Commit - | @as("ROLLBACK") Rollback - -/** Migration status */ -type migrationStatus = { - id: string, - phase: migrationPhase, - collection: string, - startedAt: string, - narrative: string, -} - -// ============================================================================= -// Request/Response Types -// ============================================================================= - -/** Query request */ -type queryRequest = { - gql: string, - provenance?: provenance, - explain?: bool, -} - -/** Create collection request */ -type createCollectionRequest = { - name: string, - @as("type") collectionType: collectionType, - schema?: JSON.t, -} - -/** Health status */ -type healthStatus = - | @as("HEALTHY") Healthy - | @as("DEGRADED") Degraded - | @as("UNHEALTHY") Unhealthy - -/** Health response */ -type healthResponse = { - status: healthStatus, - version: string, - uptimeSeconds: int, -} - -// ============================================================================= -// Error Types -// ============================================================================= - -/** API error */ -type apiError = { - code: string, - message: string, - details?: JSON.t, -} - -/** Result type for API calls */ -type result<'a> = Result.t<'a, apiError> - -// ============================================================================= -// Client Configuration -// ============================================================================= - -/** Authentication method */ -type authMethod = - | NoAuth - | ApiKey(string) - | Bearer(string) - -/** Client configuration */ -type config = { - baseUrl: string, - auth?: authMethod, - timeout?: int, - retries?: int, -} diff --git a/lithoglyph/clients/rescript/src/Lithoglyph.res b/lithoglyph/clients/rescript/src/Lithoglyph.res deleted file mode 100644 index 5b7db5d3..00000000 --- a/lithoglyph/clients/rescript/src/Lithoglyph.res +++ /dev/null @@ -1,944 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Copyright (c) 2026 Jonathan D.A. Jewell -// -// Lithoglyph ReScript Client -// Multi-protocol client for the Lithoglyph API server (REST + GraphQL) -// -// Stone-carved data for the ages: narrative-first, reversible, audit-grade database -// Compatible with Deno runtime (not Node/npm) - -open Lithoglyph_Types -open Lithoglyph_Query - -// ============================================================================= -// HTTP Helpers -// ============================================================================= - -@val external fetch: (string, 'options) => promise<'response> = "fetch" - -type fetchOptions = { - method: string, - headers: Dict.t, - body?: string, -} - -type fetchResponse = { - ok: bool, - status: int, - json: unit => promise, - text: unit => promise, -} - -// ============================================================================= -// Client -// ============================================================================= - -/** The Lithoglyph client instance */ -type t = { - config: config, -} - -/** Create a new Lithoglyph client with explicit configuration */ -let make = (~baseUrl, ~auth=?, ~timeout=?, ~retries=?, ~protocol=?) => { - config: { - baseUrl, - auth, - timeout, - retries, - protocol, - }, -} - -/** Create a client from Deno environment variables. - * Reads LITHOGLYPH_URL (or LITH_URL for backwards compatibility) - * and LITHOGLYPH_API_KEY (or LITH_API_KEY). */ -let fromEnv = () => { - let baseUrl = %raw(` - (typeof Deno !== 'undefined') - ? (Deno.env.get("LITHOGLYPH_URL") || Deno.env.get("LITH_URL") || "http://localhost:8080") - : "http://localhost:8080" - `) - let apiKey = %raw(` - (typeof Deno !== 'undefined') - ? (Deno.env.get("LITHOGLYPH_API_KEY") || Deno.env.get("LITH_API_KEY") || null) - : null - `) - - make( - ~baseUrl, - ~auth=?switch apiKey { - | Some(key) => Some(ApiKey(key)) - | None => None - }, - ) -} - -// Internal: Build headers for requests -let buildHeaders = client => { - let headers = Dict.make() - headers->Dict.set("Content-Type", "application/json") - headers->Dict.set("Accept", "application/json") - - switch client.config.auth { - | Some(ApiKey(key)) => headers->Dict.set("X-API-Key", key) - | Some(Bearer(token)) => headers->Dict.set("Authorization", `Bearer ${token}`) - | Some(NoAuth) | None => () - } - - headers -} - -// Internal: Make HTTP request to REST API -let request = async (client, ~method, ~path, ~body=?) => { - let url = `${client.config.baseUrl}${path}` - let headers = buildHeaders(client) - - let options: fetchOptions = { - method, - headers, - body: ?body->Option.map(JSON.stringify), - } - - let response: fetchResponse = await fetch(url, options) - - if !response.ok { - let text = await response.text() - Error({ - code: Int.toString(response.status), - message: text, - details: None, - }) - } else { - let json = await response.json() - Ok(json) - } -} - -// Internal: Make GraphQL request -let graphqlRequest = async (client, ~query, ~variables=?, ~operationName=?) => { - let body: JSON.t = JSON.Encode.object([ - ("query", JSON.Encode.string(query)), - ...switch variables { - | Some(v) => [("variables", v)] - | None => [] - }, - ...switch operationName { - | Some(n) => [("operationName", JSON.Encode.string(n))] - | None => [] - }, - ]) - - let url = `${client.config.baseUrl}/graphql` - let headers = buildHeaders(client) - - let options: fetchOptions = { - method: "POST", - headers, - body: ?Some(JSON.stringify(body)), - } - - let response: fetchResponse = await fetch(url, options) - - if !response.ok { - let text = await response.text() - Error({ - code: Int.toString(response.status), - message: text, - details: None, - }) - } else { - let json = await response.json() - let obj = json->JSON.Decode.object->Option.getExn - - // Check for GraphQL errors - switch obj->Dict.get("errors") { - | Some(errors) => { - let errArr = errors->JSON.Decode.array->Option.getOr([]) - let firstErr = - errArr - ->Array.get(0) - ->Option.flatMap(JSON.Decode.object) - let message = - firstErr - ->Option.flatMap(e => e->Dict.get("message")) - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("GraphQL error") - Error({ - code: "GRAPHQL_ERROR", - message, - details: Some(errors), - }) - } - | None => - switch obj->Dict.get("data") { - | Some(data) => Ok(data) - | None => - Error({ - code: "NO_DATA", - message: "GraphQL response contained no data", - details: None, - }) - } - } - } -} - -// ============================================================================= -// Query Operations (REST) -// ============================================================================= - -/** Execute an GQL query via REST API */ -let query = async (client, ~gql, ~provenance=?, ~explain=?) => { - let body: JSON.t = JSON.Encode.object([ - ("gql", JSON.Encode.string(gql)), - ...switch provenance { - | Some(p) => [ - ( - "provenance", - JSON.Encode.object([ - ("actor", JSON.Encode.string(p.actor)), - ("rationale", JSON.Encode.string(p.rationale)), - ]), - ), - ] - | None => [] - }, - ...switch explain { - | Some(true) => [("explain", JSON.Encode.bool(true))] - | Some(false) | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/query", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - rows: obj - ->Dict.get("rows") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(row => row->JSON.Decode.object->Option.getOr(Dict.make())), - rowCount: obj - ->Dict.get("rowCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - journalSeq: obj - ->Dict.get("journalSeq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - provenance: None, - timing: obj - ->Dict.get("timing") - ->Option.flatMap(JSON.Decode.object) - ->Option.map(t => { - parseMs: t->Dict.get("parseMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - planMs: t->Dict.get("planMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - executeMs: t - ->Dict.get("executeMs") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - totalMs: t->Dict.get("totalMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - }), - } - }) -} - -/** Execute a query using the query builder */ -let queryWith = async (client, builder) => { - let gql = builder->toGql - let provenance = builder.provenance - await query(client, ~gql, ~provenance?) -} - -// ============================================================================= -// Query Operations (GraphQL) -// ============================================================================= - -/** Execute an GQL query via GraphQL API */ -let queryGraphQL = async (client, ~gql, ~provenance=?) => { - let variables: JSON.t = JSON.Encode.object([ - ("gql", JSON.Encode.string(gql)), - ...switch provenance { - | Some(p) => [ - ( - "provenance", - JSON.Encode.object([ - ("actor", JSON.Encode.string(p.actor)), - ("rationale", JSON.Encode.string(p.rationale)), - ]), - ), - ] - | None => [] - }, - ]) - - let gqlQuery = ` - query ExecuteQuery($gql: String!, $provenance: ProvenanceInput) { - query(gql: $gql, provenance: $provenance) { - rows - rowCount - journalSeq - timing { - parseMs - planMs - executeMs - totalMs - } - } - } - ` - - let result = await graphqlRequest(client, ~query=gqlQuery, ~variables) - - result->Result.map(data => { - let queryObj = - data - ->JSON.Decode.object - ->Option.flatMap(d => d->Dict.get("query")) - ->Option.flatMap(JSON.Decode.object) - ->Option.getExn - { - rows: queryObj - ->Dict.get("rows") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(row => row->JSON.Decode.object->Option.getOr(Dict.make())), - rowCount: queryObj - ->Dict.get("rowCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - journalSeq: queryObj - ->Dict.get("journalSeq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - provenance: None, - timing: queryObj - ->Dict.get("timing") - ->Option.flatMap(JSON.Decode.object) - ->Option.map(t => { - parseMs: t->Dict.get("parseMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - planMs: t->Dict.get("planMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - executeMs: t - ->Dict.get("executeMs") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - totalMs: t->Dict.get("totalMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - }), - } - }) -} - -// ============================================================================= -// EXPLAIN Operations -// ============================================================================= - -/** Get the query execution plan without running the query */ -let explain = async (client, ~gql, ~analyze=?, ~verbose=?) => { - let body: JSON.t = JSON.Encode.object([ - ("gql", JSON.Encode.string(gql)), - ("explain", JSON.Encode.bool(true)), - ...switch analyze { - | Some(true) => [("analyze", JSON.Encode.bool(true))] - | Some(false) | None => [] - }, - ...switch verbose { - | Some(true) => [("verbose", JSON.Encode.bool(true))] - | Some(false) | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/query", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - let planObj = - obj - ->Dict.get("plan") - ->Option.flatMap(JSON.Decode.object) - ->Option.getExn - - { - plan: { - steps: planObj - ->Dict.get("steps") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(step => { - let s = step->JSON.Decode.object->Option.getExn - { - stepType: switch s - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("SCAN") { - | "FILTER" | "filter" => Filter - | "PROJECT" | "project" => Project - | "LIMIT" | "limit" => Limit - | "TRAVERSE" | "traverse" => Traverse - | "INSERT" | "insert" => StepInsert - | "UPDATE" | "update" => StepUpdate - | "DELETE" | "delete" => StepDelete - | _ => Scan - }, - collection: s->Dict.get("collection")->Option.flatMap(JSON.Decode.string), - expression: s->Dict.get("expression")->Option.flatMap(JSON.Decode.string), - count: s - ->Dict.get("count") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt), - details: s->Dict.get("details"), - } - }), - estimatedCost: planObj - ->Dict.get("estimatedCost") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - rationale: planObj->Dict.get("rationale")->Option.flatMap(JSON.Decode.string), - }, - timing: obj - ->Dict.get("timing") - ->Option.flatMap(JSON.Decode.object) - ->Option.map(t => { - parseMs: t->Dict.get("parseMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - planMs: t->Dict.get("planMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - executeMs: t - ->Dict.get("executeMs") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - totalMs: t->Dict.get("totalMs")->Option.flatMap(JSON.Decode.float)->Option.getOr(0.0), - }), - verboseOutput: None, - } - }) -} - -// ============================================================================= -// Collection Operations -// ============================================================================= - -/** List all collections */ -let listCollections = async client => { - let result = await request(client, ~method="GET", ~path="/v1/collections") - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - obj - ->Dict.get("collections") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(item => { - let col = item->JSON.Decode.object->Option.getExn - { - name: col->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch col - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" | "edge" => Edge - | "SCHEMA" | "schema" => Schema - | _ => Document - }, - documentCount: col - ->Dict.get("documentCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - normalForm: col->Dict.get("normalForm")->Option.flatMap(JSON.Decode.string), - schema: col->Dict.get("schema"), - } - }) - }) -} - -/** Get a specific collection by name */ -let getCollection = async (client, ~name) => { - let result = await request(client, ~method="GET", ~path=`/v1/collections/${name}`) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - name: obj->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch obj - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" | "edge" => Edge - | "SCHEMA" | "schema" => Schema - | _ => Document - }, - documentCount: obj - ->Dict.get("documentCount") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - normalForm: obj->Dict.get("normalForm")->Option.flatMap(JSON.Decode.string), - schema: obj->Dict.get("schema"), - } - }) -} - -/** Create a new collection */ -let createCollection = async (client, ~name, ~collectionType=Document, ~schema=?) => { - let typeStr = switch collectionType { - | Document => "DOCUMENT" - | Edge => "EDGE" - | Schema => "SCHEMA" - } - - let body: JSON.t = JSON.Encode.object([ - ("name", JSON.Encode.string(name)), - ("type", JSON.Encode.string(typeStr)), - ...switch schema { - | Some(s) => [("schema", s)] - | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/collections", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - name: obj->Dict.get("name")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collectionType: switch obj - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DOCUMENT") { - | "EDGE" | "edge" => Edge - | "SCHEMA" | "schema" => Schema - | _ => Document - }, - documentCount: 0, - normalForm: obj->Dict.get("normalForm")->Option.flatMap(JSON.Decode.string), - schema: obj->Dict.get("schema"), - } - }) -} - -/** Delete a collection */ -let deleteCollection = async (client, ~name) => { - let result = await request(client, ~method="DELETE", ~path=`/v1/collections/${name}`) - result->Result.map(_ => ()) -} - -// ============================================================================= -// Journal Operations -// ============================================================================= - -/** Get journal entries with optional filtering */ -let getJournal = async (client, ~since=?, ~limit=?, ~collection=?) => { - let params = [] - switch since { - | Some(seq) => params->Array.push(`since=${Int.toString(seq)}`)->ignore - | None => () - } - switch limit { - | Some(n) => params->Array.push(`limit=${Int.toString(n)}`)->ignore - | None => () - } - switch collection { - | Some(c) => params->Array.push(`collection=${c}`)->ignore - | None => () - } - - let queryStr = params->Array.length > 0 ? `?${params->Array.join("&")}` : "" - - let result = await request(client, ~method="GET", ~path=`/v1/journal${queryStr}`) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - entries: obj - ->Dict.get("entries") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(item => { - let entry = item->JSON.Decode.object->Option.getExn - { - seq: entry - ->Dict.get("seq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - timestamp: entry - ->Dict.get("timestamp") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - operation: switch entry - ->Dict.get("operation") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("insert") { - | "update" | "UPDATE" => Update - | "delete" | "DELETE" => Delete - | "CREATE_COLLECTION" | "create_collection" => CreateCollection - | "DROP_COLLECTION" | "drop_collection" => DropCollection - | "MIGRATION_START" | "migration_start" => MigrationStart - | "MIGRATION_COMMIT" | "migration_commit" => MigrationCommit - | _ => Insert - }, - collection: entry->Dict.get("collection")->Option.flatMap(JSON.Decode.string), - documentId: entry->Dict.get("documentId")->Option.flatMap(JSON.Decode.string), - before: entry->Dict.get("before"), - after: entry->Dict.get("after"), - provenance: entry - ->Dict.get("provenance") - ->Option.flatMap(JSON.Decode.object) - ->Option.map(p => { - actor: p - ->Dict.get("actor") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - rationale: p - ->Dict.get("rationale") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - }), - inverse: entry->Dict.get("inverse")->Option.flatMap(JSON.Decode.string), - } - }), - hasMore: obj - ->Dict.get("hasMore") - ->Option.flatMap(JSON.Decode.bool) - ->Option.getOr(false), - nextSeq: obj - ->Dict.get("nextSeq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt), - } - }) -} - -// ============================================================================= -// Normalization Operations -// ============================================================================= - -/** Discover functional dependencies in a collection */ -let discoverDependencies = async (client, ~collection, ~confidence=?) => { - let body: JSON.t = JSON.Encode.object([ - ("collection", JSON.Encode.string(collection)), - ...switch confidence { - | Some(c) => [("confidenceThreshold", JSON.Encode.float(c))] - | None => [] - }, - ]) - - let result = await request(client, ~method="POST", ~path="/v1/normalize/discover", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - collection: obj - ->Dict.get("collection") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - functionalDependencies: obj - ->Dict.get("functionalDependencies") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(item => { - let fd = item->JSON.Decode.object->Option.getExn - { - determinant: fd - ->Dict.get("determinant") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - dependent: fd - ->Dict.get("dependent") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - confidence: fd - ->Dict.get("confidence") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - tier: switch fd - ->Dict.get("tier") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("medium") { - | "high" | "HIGH" => High - | "low" | "LOW" => Low - | _ => Medium - }, - } - }), - candidateKeys: obj - ->Dict.get("candidateKeys") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(keyArr => - keyArr - ->JSON.Decode.array - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string) - ), - } - }) -} - -/** Analyze normal form of a collection */ -let analyzeNormalForm = async (client, ~collection) => { - let body: JSON.t = JSON.Encode.object([("collection", JSON.Encode.string(collection))]) - - let result = await request(client, ~method="POST", ~path="/v1/normalize/analyze", ~body) - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - collection: obj - ->Dict.get("collection") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - currentForm: switch obj - ->Dict.get("currentForm") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("1NF") { - | "2NF" => NF2 - | "3NF" => NF3 - | "BCNF" => BCNF - | _ => NF1 - }, - violations: obj - ->Dict.get("violations") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(item => { - let v = item->JSON.Decode.object->Option.getExn - { - violationType: switch v - ->Dict.get("type") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("TRANSITIVE_DEPENDENCY") { - | "PARTIAL_DEPENDENCY" | "partial_dependency" => PartialDependency - | "BCNF_VIOLATION" | "bcnf_violation" => BcnfViolation - | _ => TransitiveDependency - }, - description: v - ->Dict.get("description") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - affectedFields: v - ->Dict.get("affectedFields") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - } - }), - recommendations: obj - ->Dict.get("recommendations") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.map(item => { - let r = item->JSON.Decode.object->Option.getExn - { - action: switch r - ->Dict.get("action") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("DECOMPOSE") { - | "ADD_CONSTRAINT" | "add_constraint" => AddConstraint - | "DENORMALIZE" | "denormalize" => Denormalize - | _ => Decompose - }, - description: r - ->Dict.get("description") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - targetForm: r - ->Dict.get("targetForm") - ->Option.flatMap(JSON.Decode.string) - ->Option.map(f => - switch f { - | "1NF" => NF1 - | "2NF" => NF2 - | "3NF" => NF3 - | _ => BCNF - } - ), - migrationSteps: r - ->Dict.get("migrationSteps") - ->Option.flatMap(JSON.Decode.array) - ->Option.getOr([]) - ->Array.filterMap(JSON.Decode.string), - } - }), - } - }) -} - -// ============================================================================= -// Migration Operations -// ============================================================================= - -// Internal: Parse migration response (shared between start/shadow/commit/abort) -let parseMigrationResponse = json => { - let obj = json->JSON.Decode.object->Option.getExn - { - id: obj->Dict.get("id")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - collection: obj - ->Dict.get("collection") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - phase: switch obj - ->Dict.get("phase") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("announce") { - | "shadow" | "SHADOW" => Shadow - | "commit" | "COMMIT" => Commit - | "complete" | "COMPLETE" => Complete - | "aborted" | "ABORTED" => Aborted - | "rollback" | "ROLLBACK" => Rollback - | _ => Announce - }, - startedAt: obj - ->Dict.get("startedAt") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - narrative: obj - ->Dict.get("narrative") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - } -} - -/** Start a schema migration */ -let startMigration = async (client, ~collection, ~targetForm) => { - let targetStr = switch targetForm { - | NF1 => "1NF" - | NF2 => "2NF" - | NF3 => "3NF" - | BCNF => "BCNF" - } - - let body: JSON.t = JSON.Encode.object([ - ("collection", JSON.Encode.string(collection)), - ("targetForm", JSON.Encode.string(targetStr)), - ]) - - let result = await request(client, ~method="POST", ~path="/v1/migrate/start", ~body) - result->Result.map(parseMigrationResponse) -} - -/** Advance a migration to the shadow (dual-write) phase */ -let advanceToShadow = async (client, ~migrationId) => { - let body: JSON.t = JSON.Encode.object([("migrationId", JSON.Encode.string(migrationId))]) - let result = await request(client, ~method="POST", ~path="/v1/migrate/shadow", ~body) - result->Result.map(parseMigrationResponse) -} - -/** Commit a migration (finalize the schema change) */ -let commitMigration = async (client, ~migrationId) => { - let body: JSON.t = JSON.Encode.object([("migrationId", JSON.Encode.string(migrationId))]) - let result = await request(client, ~method="POST", ~path="/v1/migrate/commit", ~body) - result->Result.map(parseMigrationResponse) -} - -/** Abort a migration (roll back to the original schema) */ -let abortMigration = async (client, ~migrationId) => { - let body: JSON.t = JSON.Encode.object([("migrationId", JSON.Encode.string(migrationId))]) - let result = await request(client, ~method="POST", ~path="/v1/migrate/abort", ~body) - result->Result.map(parseMigrationResponse) -} - -// ============================================================================= -// Health Check -// ============================================================================= - -/** Check server health via REST API */ -let health = async client => { - let result = await request(client, ~method="GET", ~path="/v1/health") - - result->Result.map(json => { - let obj = json->JSON.Decode.object->Option.getExn - { - status: switch obj - ->Dict.get("status") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("UNHEALTHY") { - | "healthy" | "HEALTHY" => Healthy - | "degraded" | "DEGRADED" => Degraded - | _ => Unhealthy - }, - version: obj->Dict.get("version")->Option.flatMap(JSON.Decode.string)->Option.getOr(""), - uptimeSeconds: obj - ->Dict.get("uptime") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - checks: obj - ->Dict.get("checks") - ->Option.flatMap(JSON.Decode.object) - ->Option.map(checks => { - let result = [] - checks - ->Dict.toArray - ->Array.forEach(((name, value)) => { - let status = switch value->JSON.Decode.string->Option.getOr("fail") { - | "pass" | "PASS" => Pass - | _ => Fail - } - result->Array.push({name, status})->ignore - }) - result - }), - } - }) -} - -/** Check server health via GraphQL API */ -let healthGraphQL = async client => { - let gqlQuery = ` - query Health { - health { - status - version - uptimeSeconds - checks { - name - status - } - } - } - ` - - let result = await graphqlRequest(client, ~query=gqlQuery) - - result->Result.map(data => { - let healthObj = - data - ->JSON.Decode.object - ->Option.flatMap(d => d->Dict.get("health")) - ->Option.flatMap(JSON.Decode.object) - ->Option.getExn - { - status: switch healthObj - ->Dict.get("status") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("UNHEALTHY") { - | "HEALTHY" => Healthy - | "DEGRADED" => Degraded - | _ => Unhealthy - }, - version: healthObj - ->Dict.get("version") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - uptimeSeconds: healthObj - ->Dict.get("uptimeSeconds") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - checks: None, - } - }) -} - -// ============================================================================= -// Re-exports -// ============================================================================= - -module Types = Lithoglyph_Types -module Query = Lithoglyph_Query -module Subscriptions = Lithoglyph_Subscriptions diff --git a/lithoglyph/clients/rescript/src/Lithoglyph_Query.res b/lithoglyph/clients/rescript/src/Lithoglyph_Query.res deleted file mode 100644 index 71ca3510..00000000 --- a/lithoglyph/clients/rescript/src/Lithoglyph_Query.res +++ /dev/null @@ -1,397 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Copyright (c) 2026 Jonathan D.A. Jewell -// -// Lithoglyph ReScript Client - Query Builder -// Type-safe GQL query construction with fluent API -// -// Compatible with Deno runtime (not Node/npm) - -open Lithoglyph_Types - -// ============================================================================= -// Comparison Operators -// ============================================================================= - -/** Comparison operators for GQL WHERE clauses */ -type compareOp = - | Eq - | Ne - | Lt - | Le - | Gt - | Ge - | Like - | In - -/** Convert a comparison operator to its GQL string representation */ -let compareOpToString = op => - switch op { - | Eq => "=" - | Ne => "!=" - | Lt => "<" - | Le => "<=" - | Gt => ">" - | Ge => ">=" - | Like => "LIKE" - | In => "IN" - } - -// ============================================================================= -// Filter Expressions -// ============================================================================= - -/** Recursive filter expression type for building complex WHERE clauses */ -type rec filterExpr = - | Field(string, compareOp, JSON.t) - | And(filterExpr, filterExpr) - | Or(filterExpr, filterExpr) - | Not(filterExpr) - -/** Convert a filter expression to its GQL string representation */ -let rec filterToGql = filter => - switch filter { - | Field(name, op, value) => { - let valueStr = switch value { - | JSON.String(s) => `"${s}"` - | JSON.Number(n) => Float.toString(n) - | JSON.Boolean(b) => b ? "true" : "false" - | JSON.Null => "null" - | _ => JSON.stringify(value) - } - `${name} ${compareOpToString(op)} ${valueStr}` - } - | And(a, b) => `(${filterToGql(a)} AND ${filterToGql(b)})` - | Or(a, b) => `(${filterToGql(a)} OR ${filterToGql(b)})` - | Not(f) => `NOT (${filterToGql(f)})` - } - -// ============================================================================= -// SELECT Query Builder -// ============================================================================= - -/** Query builder for constructing SELECT statements */ -type queryBuilder = { - mutable collection: option, - mutable fields: option>, - mutable filter: option, - mutable limit: option, - mutable offset: option, - mutable orderBy: option<(string, bool)>, - mutable provenance: option, -} - -/** Create a new SELECT query builder */ -let make = () => { - collection: None, - fields: None, - filter: None, - limit: None, - offset: None, - orderBy: None, - provenance: None, -} - -/** Set the collection to query from */ -let from = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -/** Set the fields to select (default: all) */ -let select = (builder, fieldList) => { - builder.fields = Some(fieldList) - builder -} - -/** Add a WHERE filter expression */ -let where = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -/** Add a field comparison filter (convenience, ANDs with existing) */ -let whereField = (builder, fieldName, op, value) => { - let newFilter = Field(fieldName, op, value) - builder.filter = switch builder.filter { - | Some(existing) => Some(And(existing, newFilter)) - | None => Some(newFilter) - } - builder -} - -/** Set the maximum number of rows to return */ -let limit = (builder, n) => { - builder.limit = Some(n) - builder -} - -/** Set the number of rows to skip */ -let offset = (builder, n) => { - builder.offset = Some(n) - builder -} - -/** Set the ordering field and direction */ -let orderBy = (builder, field, ~ascending=true) => { - builder.orderBy = Some((field, ascending)) - builder -} - -/** Attach provenance metadata to the query */ -let withProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -/** Build the GQL query string from the query builder */ -let toGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let fieldsStr = switch builder.fields { - | Some(fields) => fields->Array.join(", ") - | None => "*" - } - - let mut query = `SELECT ${fieldsStr} FROM ${collection}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.orderBy { - | Some((field, asc)) => { - let dir = asc ? "ASC" : "DESC" - query = query ++ ` ORDER BY ${field} ${dir}` - } - | None => () - } - - switch builder.limit { - | Some(n) => query = query ++ ` LIMIT ${Int.toString(n)}` - | None => () - } - - switch builder.offset { - | Some(n) => query = query ++ ` OFFSET ${Int.toString(n)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// INSERT Builder -// ============================================================================= - -/** Builder for constructing INSERT statements */ -type insertBuilder = { - mutable collection: option, - mutable document: option, - mutable provenance: option, -} - -/** Create a new INSERT builder */ -let makeInsert = () => { - collection: None, - document: None, - provenance: None, -} - -/** Set the target collection for the insert */ -let into = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -/** Set the document to insert */ -let values = (builder, doc) => { - builder.document = Some(doc) - builder -} - -/** Attach provenance metadata to the insert */ -let insertWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -/** Build the GQL INSERT string */ -let insertToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let doc = switch builder.document { - | Some(d) => JSON.stringify(d) - | None => panic("Document is required") - } - - let mut query = `INSERT INTO ${collection} ${doc}` - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// UPDATE Builder -// ============================================================================= - -/** Builder for constructing UPDATE statements */ -type updateBuilder = { - mutable collection: option, - mutable sets: array<(string, JSON.t)>, - mutable filter: option, - mutable provenance: option, -} - -/** Create a new UPDATE builder */ -let makeUpdate = () => { - collection: None, - sets: [], - filter: None, - provenance: None, -} - -/** Set the target collection for the update */ -let updateCollection = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -/** Add a SET clause to the update */ -let set = (builder, field, value) => { - builder.sets = builder.sets->Array.concat([(field, value)]) - builder -} - -/** Add a WHERE filter to the update */ -let updateWhere = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -/** Attach provenance metadata to the update */ -let updateWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -/** Build the GQL UPDATE string */ -let updateToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - if builder.sets->Array.length == 0 { - panic("At least one SET clause is required") - } - - let setsClauses = - builder.sets - ->Array.map(((field, value)) => { - let valueStr = switch value { - | JSON.String(s) => `"${s}"` - | JSON.Number(n) => Float.toString(n) - | JSON.Boolean(b) => b ? "true" : "false" - | JSON.Null => "null" - | _ => JSON.stringify(value) - } - `${field} = ${valueStr}` - }) - ->Array.join(", ") - - let mut query = `UPDATE ${collection} SET ${setsClauses}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} - -// ============================================================================= -// DELETE Builder -// ============================================================================= - -/** Builder for constructing DELETE statements */ -type deleteBuilder = { - mutable collection: option, - mutable filter: option, - mutable provenance: option, -} - -/** Create a new DELETE builder */ -let makeDelete = () => { - collection: None, - filter: None, - provenance: None, -} - -/** Set the target collection for the delete */ -let deleteFrom = (builder, collectionName) => { - builder.collection = Some(collectionName) - builder -} - -/** Add a WHERE filter to the delete */ -let deleteWhere = (builder, filter) => { - builder.filter = Some(filter) - builder -} - -/** Attach provenance metadata to the delete */ -let deleteWithProvenance = (builder, prov) => { - builder.provenance = Some(prov) - builder -} - -/** Build the GQL DELETE string */ -let deleteToGql = builder => { - let collection = switch builder.collection { - | Some(c) => c - | None => panic("Collection is required") - } - - let mut query = `DELETE FROM ${collection}` - - switch builder.filter { - | Some(filter) => query = query ++ ` WHERE ${filterToGql(filter)}` - | None => () - } - - switch builder.provenance { - | Some(prov) => - query = - query ++ ` WITH PROVENANCE { actor: "${prov.actor}", rationale: "${prov.rationale}" }` - | None => () - } - - query -} diff --git a/lithoglyph/clients/rescript/src/Lithoglyph_Subscriptions.res b/lithoglyph/clients/rescript/src/Lithoglyph_Subscriptions.res deleted file mode 100644 index a2cae711..00000000 --- a/lithoglyph/clients/rescript/src/Lithoglyph_Subscriptions.res +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Copyright (c) 2026 Jonathan D.A. Jewell -// -// Lithoglyph ReScript Client - WebSocket Subscriptions -// Real-time journal streaming and migration progress via graphql-ws protocol -// -// Compatible with Deno runtime (not Node/npm) - -open Lithoglyph_Types - -// ============================================================================= -// WebSocket FFI -// ============================================================================= - -/** Deno/browser WebSocket binding */ -type webSocket - -@new external createWebSocket: (string, ~protocols: array=?) => webSocket = "WebSocket" - -@set external onOpen: (webSocket, unit => unit) => unit = "onopen" -@set external onClose: (webSocket, 'event => unit) => unit = "onclose" -@set external onError: (webSocket, 'event => unit) => unit = "onerror" -@set external onMessage: (webSocket, 'event => unit) => unit = "onmessage" -@send external wsSend: (webSocket, string) => unit = "send" -@send external wsClose: (webSocket, ~code: int=?, ~reason: string=?) => unit = "close" - -// ============================================================================= -// graphql-ws Protocol Messages -// ============================================================================= - -/** Create a connection_init message for the graphql-ws protocol */ -let connectionInitMsg = () => JSON.stringify(JSON.Encode.object([ - ("type", JSON.Encode.string("connection_init")), -])) - -/** Create a subscribe message for the graphql-ws protocol */ -let subscribeMsg = (~id, ~query, ~variables=?) => { - JSON.stringify(JSON.Encode.object([ - ("id", JSON.Encode.string(id)), - ("type", JSON.Encode.string("subscribe")), - ("payload", JSON.Encode.object([ - ("query", JSON.Encode.string(query)), - ...switch variables { - | Some(v) => [("variables", v)] - | None => [] - }, - ])), - ])) -} - -/** Create a complete (unsubscribe) message */ -let completeMsg = (~id) => { - JSON.stringify(JSON.Encode.object([ - ("id", JSON.Encode.string(id)), - ("type", JSON.Encode.string("complete")), - ])) -} - -// ============================================================================= -// Subscription Client -// ============================================================================= - -/** Subscription handle returned by subscribe operations */ -type subscriptionHandle = { - id: string, - unsubscribe: unit => unit, -} - -/** Convert an HTTP base URL to the corresponding WebSocket URL for GraphQL */ -let wsUrlFromHttp = baseUrl => { - let wsUrl = if baseUrl->String.startsWith("https://") { - "wss://" ++ baseUrl->String.sliceToEnd(~start=8) - } else if baseUrl->String.startsWith("http://") { - "ws://" ++ baseUrl->String.sliceToEnd(~start=7) - } else { - baseUrl - } - `${wsUrl}/graphql` -} - -/** Subscribe to journal entries in real time. - * Returns a subscription handle; call handle.unsubscribe() to stop. - * The onEntry callback fires for each new journal entry received. */ -let subscribeJournal = ( - ~baseUrl, - ~auth=?, - ~collection=?, - ~since=?, - ~onEntry, - ~onError=?, - ~onComplete=?, -) => { - ignore(auth) - let wsUrl = wsUrlFromHttp(baseUrl) - let ws = createWebSocket(wsUrl, ~protocols=["graphql-ws"]) - let subId = "journal-stream-1" - - onOpen(ws, () => { - wsSend(ws, connectionInitMsg()) - - let variables = JSON.Encode.object([ - ...switch collection { - | Some(c) => [("collection", JSON.Encode.string(c))] - | None => [] - }, - ...switch since { - | Some(s) => [("since", JSON.Encode.int(s))] - | None => [] - }, - ]) - - let query = ` - subscription JournalStream($collection: String, $since: BigInt) { - journalStream(collection: $collection, since: $since) { - seq - timestamp - operation - collection - documentId - } - } - ` - wsSend(ws, subscribeMsg(~id=subId, ~query, ~variables)) - }) - - onMessage(ws, event => { - let data: string = %raw(`event.data`) - let parsed = data->JSON.parseExn->JSON.Decode.object - switch parsed { - | Some(msg) => { - let msgType = msg->Dict.get("type")->Option.flatMap(JSON.Decode.string)->Option.getOr("") - switch msgType { - | "next" => { - let payload = - msg - ->Dict.get("payload") - ->Option.flatMap(JSON.Decode.object) - ->Option.flatMap(p => p->Dict.get("data")) - ->Option.flatMap(JSON.Decode.object) - ->Option.flatMap(d => d->Dict.get("journalStream")) - ->Option.flatMap(JSON.Decode.object) - switch payload { - | Some(entry) => - onEntry({ - seq: entry - ->Dict.get("seq") - ->Option.flatMap(JSON.Decode.float) - ->Option.map(Float.toInt) - ->Option.getOr(0), - timestamp: entry - ->Dict.get("timestamp") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - operation: switch entry - ->Dict.get("operation") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("INSERT") { - | "UPDATE" => Update - | "DELETE" => Delete - | _ => Insert - }, - collection: entry->Dict.get("collection")->Option.flatMap(JSON.Decode.string), - documentId: entry->Dict.get("documentId")->Option.flatMap(JSON.Decode.string), - }) - | None => () - } - } - | "error" => { - let errMsg = - msg - ->Dict.get("payload") - ->Option.map(JSON.stringify) - ->Option.getOr("Subscription error") - switch onError { - | Some(handler) => handler(errMsg) - | None => () - } - } - | "complete" => - switch onComplete { - | Some(handler) => handler() - | None => () - } - | _ => () - } - } - | None => () - } - }) - - onError(ws, _event => { - switch onError { - | Some(handler) => handler("WebSocket connection error") - | None => () - } - }) - - { - id: subId, - unsubscribe: () => { - wsSend(ws, completeMsg(~id=subId)) - wsClose(ws) - }, - } -} - -/** Subscribe to migration progress updates in real time. - * Returns a subscription handle; call handle.unsubscribe() to stop. */ -let subscribeMigrationProgress = ( - ~baseUrl, - ~auth=?, - ~migrationId, - ~onProgress, - ~onError=?, - ~onComplete=?, -) => { - ignore(auth) - let wsUrl = wsUrlFromHttp(baseUrl) - let ws = createWebSocket(wsUrl, ~protocols=["graphql-ws"]) - let subId = `migration-progress-${migrationId}` - - onOpen(ws, () => { - wsSend(ws, connectionInitMsg()) - - let variables = JSON.Encode.object([ - ("migrationId", JSON.Encode.string(migrationId)), - ]) - - let query = ` - subscription MigrationProgress($migrationId: ID!) { - migrationProgress(migrationId: $migrationId) { - migrationId - phase - progress - message - } - } - ` - wsSend(ws, subscribeMsg(~id=subId, ~query, ~variables)) - }) - - onMessage(ws, event => { - let data: string = %raw(`event.data`) - let parsed = data->JSON.parseExn->JSON.Decode.object - switch parsed { - | Some(msg) => { - let msgType = msg->Dict.get("type")->Option.flatMap(JSON.Decode.string)->Option.getOr("") - switch msgType { - | "next" => { - let payload = - msg - ->Dict.get("payload") - ->Option.flatMap(JSON.Decode.object) - ->Option.flatMap(p => p->Dict.get("data")) - ->Option.flatMap(JSON.Decode.object) - ->Option.flatMap(d => d->Dict.get("migrationProgress")) - ->Option.flatMap(JSON.Decode.object) - switch payload { - | Some(progress) => - onProgress({ - migrationId: progress - ->Dict.get("migrationId") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - phase: switch progress - ->Dict.get("phase") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr("ANNOUNCE") { - | "SHADOW" => Shadow - | "COMMIT" => Commit - | "COMPLETE" => Complete - | "ABORTED" => Aborted - | _ => Announce - }, - progress: progress - ->Dict.get("progress") - ->Option.flatMap(JSON.Decode.float) - ->Option.getOr(0.0), - message: progress - ->Dict.get("message") - ->Option.flatMap(JSON.Decode.string) - ->Option.getOr(""), - }) - | None => () - } - } - | "error" => { - let errMsg = - msg - ->Dict.get("payload") - ->Option.map(JSON.stringify) - ->Option.getOr("Subscription error") - switch onError { - | Some(handler) => handler(errMsg) - | None => () - } - } - | "complete" => - switch onComplete { - | Some(handler) => handler() - | None => () - } - | _ => () - } - } - | None => () - } - }) - - onError(ws, _event => { - switch onError { - | Some(handler) => handler("WebSocket connection error") - | None => () - } - }) - - { - id: subId, - unsubscribe: () => { - wsSend(ws, completeMsg(~id=subId)) - wsClose(ws) - }, - } -} diff --git a/lithoglyph/clients/rescript/src/Lithoglyph_Types.res b/lithoglyph/clients/rescript/src/Lithoglyph_Types.res deleted file mode 100644 index 9b8e9528..00000000 --- a/lithoglyph/clients/rescript/src/Lithoglyph_Types.res +++ /dev/null @@ -1,327 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -// Copyright (c) 2026 Jonathan D.A. Jewell -// -// Lithoglyph ReScript Client - Type Definitions -// Stone-carved data for the ages: narrative-first, reversible, audit-grade database -// -// Compatible with Deno runtime (not Node/npm) - -// ============================================================================= -// Core Types -// ============================================================================= - -/** Provenance metadata for audit trail */ -type provenance = { - actor: string, - rationale: string, - timestamp?: string, - source?: string, -} - -/** Query result row */ -type row = Dict.t - -/** Timing information for query execution */ -type timing = { - parseMs: float, - planMs: float, - executeMs: float, - totalMs: float, -} - -/** Query result from GQL execution */ -type queryResult = { - rows: array, - rowCount: int, - journalSeq: int, - provenance?: provenance, - timing?: timing, -} - -/** Collection type */ -type collectionType = - | @as("DOCUMENT") Document - | @as("EDGE") Edge - | @as("SCHEMA") Schema - -/** Collection metadata */ -type collection = { - name: string, - @as("type") collectionType: collectionType, - documentCount: int, - normalForm?: string, - schema?: JSON.t, -} - -/** Journal operation type */ -type journalOperation = - | @as("INSERT") Insert - | @as("UPDATE") Update - | @as("DELETE") Delete - | @as("CREATE_COLLECTION") CreateCollection - | @as("DROP_COLLECTION") DropCollection - | @as("MIGRATION_START") MigrationStart - | @as("MIGRATION_COMMIT") MigrationCommit - -/** Journal entry */ -type journalEntry = { - seq: int, - timestamp: string, - operation: journalOperation, - collection?: string, - documentId?: string, - before?: JSON.t, - after?: JSON.t, - provenance?: provenance, - inverse?: string, -} - -/** Journal response with pagination */ -type journalResponse = { - entries: array, - hasMore: bool, - nextSeq?: int, -} - -// ============================================================================= -// Normalization Types -// ============================================================================= - -/** Normal form level */ -type normalForm = - | @as("1NF") NF1 - | @as("2NF") NF2 - | @as("3NF") NF3 - | @as("BCNF") BCNF - -/** Confidence tier for discovered dependencies */ -type confidenceTier = - | @as("HIGH") High - | @as("MEDIUM") Medium - | @as("LOW") Low - -/** Functional dependency */ -type functionalDependency = { - determinant: array, - dependent: string, - confidence: float, - tier: confidenceTier, -} - -/** Discover result with FDs and candidate keys */ -type discoverResult = { - collection: string, - functionalDependencies: array, - candidateKeys: array>, -} - -/** Violation type */ -type violationType = - | @as("PARTIAL_DEPENDENCY") PartialDependency - | @as("TRANSITIVE_DEPENDENCY") TransitiveDependency - | @as("BCNF_VIOLATION") BcnfViolation - -/** Schema violation */ -type violation = { - @as("type") violationType: violationType, - description: string, - affectedFields: array, -} - -/** Recommendation action */ -type recommendationAction = - | @as("DECOMPOSE") Decompose - | @as("ADD_CONSTRAINT") AddConstraint - | @as("DENORMALIZE") Denormalize - -/** Schema recommendation */ -type recommendation = { - action: recommendationAction, - description: string, - targetForm?: normalForm, - migrationSteps: array, -} - -/** Normal form analysis result */ -type analyzeResult = { - collection: string, - currentForm: normalForm, - violations: array, - recommendations: array, -} - -// ============================================================================= -// Migration Types -// ============================================================================= - -/** Migration phase */ -type migrationPhase = - | @as("ANNOUNCE") Announce - | @as("SHADOW") Shadow - | @as("COMMIT") Commit - | @as("COMPLETE") Complete - | @as("ABORTED") Aborted - | @as("ROLLBACK") Rollback - -/** Migration status */ -type migrationStatus = { - id: string, - collection: string, - phase: migrationPhase, - startedAt: string, - narrative: string, -} - -/** Migration progress (for WebSocket subscription) */ -type migrationProgress = { - migrationId: string, - phase: migrationPhase, - progress: float, - message: string, -} - -// ============================================================================= -// EXPLAIN Types -// ============================================================================= - -/** Plan step type */ -type stepType = - | @as("SCAN") Scan - | @as("FILTER") Filter - | @as("PROJECT") Project - | @as("LIMIT") Limit - | @as("TRAVERSE") Traverse - | @as("INSERT") StepInsert - | @as("UPDATE") StepUpdate - | @as("DELETE") StepDelete - -/** Query plan step */ -type planStep = { - @as("type") stepType: stepType, - collection?: string, - expression?: string, - count?: int, - details?: JSON.t, -} - -/** Query plan */ -type queryPlan = { - steps: array, - estimatedCost: float, - rationale?: string, -} - -/** EXPLAIN result */ -type explainResult = { - plan: queryPlan, - timing?: timing, - verboseOutput?: string, -} - -// ============================================================================= -// Health Types -// ============================================================================= - -/** Health status */ -type healthStatus = - | @as("HEALTHY") Healthy - | @as("DEGRADED") Degraded - | @as("UNHEALTHY") Unhealthy - -/** Check status */ -type checkStatus = - | @as("PASS") Pass - | @as("FAIL") Fail - -/** Health check */ -type healthCheck = { - name: string, - status: checkStatus, -} - -/** Health response */ -type healthResponse = { - status: healthStatus, - version: string, - uptimeSeconds: int, - checks?: array, -} - -// ============================================================================= -// Error Types -// ============================================================================= - -/** API error */ -type apiError = { - code: string, - message: string, - details?: JSON.t, -} - -/** Result type for API calls */ -type result<'a> = Result.t<'a, apiError> - -// ============================================================================= -// Client Configuration -// ============================================================================= - -/** Authentication method */ -type authMethod = - | NoAuth - | ApiKey(string) - | Bearer(string) - -/** Protocol to use when communicating with the server */ -type protocol = - | REST - | GraphQL - -/** Client configuration */ -type config = { - baseUrl: string, - auth?: authMethod, - timeout?: int, - retries?: int, - protocol?: protocol, -} - -// ============================================================================= -// GraphQL Types -// ============================================================================= - -/** GraphQL request */ -type graphqlRequest = { - query: string, - operationName?: string, - variables?: JSON.t, -} - -/** GraphQL error */ -type graphqlError = { - message: string, - locations?: array<{line: int, column: int}>, - path?: array, -} - -/** GraphQL response */ -type graphqlResponse = { - data?: JSON.t, - errors?: array, -} - -// ============================================================================= -// WebSocket / Subscription Types -// ============================================================================= - -/** Subscription type for real-time streaming */ -type subscriptionType = - | JournalStream - | QueryStream - | MigrationProgressStream - -/** Subscription message from the server */ -type subscriptionMessage<'a> = { - @as("type") msgType: string, - id?: string, - payload?: 'a, -} diff --git a/lithoglyph/contractiles/README.adoc b/lithoglyph/contractiles/README.adoc deleted file mode 100644 index d19a3877..00000000 --- a/lithoglyph/contractiles/README.adoc +++ /dev/null @@ -1,19 +0,0 @@ -= Contractiles Template Set -:toc: -:sectnums: - -This directory contains the generalized contractiles templates. Copy the `contractiles/` directory into a new repo to establish a consistent operational, validation, trust, recovery, and intent framework. - -== Fill-In Instructions - -1. Update the Mustfile to reflect your real invariants (paths, schema versions, ports). -2. Replace Trustfile.hs placeholders with your actual key paths and verification commands. -3. Adjust Dustfile handlers to match your rollback and recovery tooling. -4. Update Intentfile to mirror the roadmap you want the system to evolve toward. - -== Contents - -* `must/Mustfile` - required invariants and validations. -* `trust/Trustfile.hs` - cryptographic verification steps. -* `dust/Dustfile` - rollback and recovery semantics. -* `lust/Intentfile` - future intent and roadmap direction. diff --git a/lithoglyph/contractiles/dust/Dustfile b/lithoglyph/contractiles/dust/Dustfile deleted file mode 100644 index aece7295..00000000 --- a/lithoglyph/contractiles/dust/Dustfile +++ /dev/null @@ -1 +0,0 @@ -content of dustfile diff --git a/lithoglyph/contractiles/must/Mustfile b/lithoglyph/contractiles/must/Mustfile deleted file mode 100644 index ee751099..00000000 --- a/lithoglyph/contractiles/must/Mustfile +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Mustfile - mandatory checks -# See: https://github.com/hyperpolymath/mustfile - -version: 1 - -checks: - - name: security - run: just lint - - name: tests - run: just test - - name: format - run: just fmt - diff --git a/lithoglyph/contractiles/trust/Trustfile b/lithoglyph/contractiles/trust/Trustfile deleted file mode 100644 index abf38c03..00000000 --- a/lithoglyph/contractiles/trust/Trustfile +++ /dev/null @@ -1 +0,0 @@ -contents of Trustfile here diff --git a/lithoglyph/core-factor/README.adoc b/lithoglyph/core-factor/README.adoc deleted file mode 100644 index a7dc5dfb..00000000 --- a/lithoglyph/core-factor/README.adoc +++ /dev/null @@ -1,24 +0,0 @@ -= Form.Runtime (Factor) -:toc: - -The query runtime, implemented in Factor. - -== Purpose - -* GQL parse/plan/exec -* Query explanation (EXPLAIN) -* Schema/constraint introspection -* Provenance surfaces - -== GQL PoC Subset - -* `INSERT` document into collection -* `INSERT` edge (from, to, type, props) -* `SELECT` with simple predicates -* `EXPLAIN` (returns plan + reasons) -* `INTROSPECT` schema/constraints -* Optional provenance output - -== Status - -Implementation pending GQL grammar specification (Q-GQL-POC-001). diff --git a/lithoglyph/core-factor/gql-c-api/gql-c-api.factor b/lithoglyph/core-factor/gql-c-api/gql-c-api.factor deleted file mode 100644 index d150b574..00000000 --- a/lithoglyph/core-factor/gql-c-api/gql-c-api.factor +++ /dev/null @@ -1,181 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! gql-c-api - C API for GQL Runtime -! -! Exports C-callable functions for integration with Zig FFI - -USING: accessors alien alien.c-types alien.data alien.strings arrays -assocs byte-arrays classes.struct combinators continuations gql -formatting hashtables io io.encodings.utf8 json json.writer kernel -locals math namespaces sequences splitting storage-backend strings ; - -IN: gql-c-api - -! ============================================================ -! C API Structures -! ============================================================ - -STRUCT: c-string-result - { data char* } - { length size_t } - { status int32_t } ; - -! Status codes matching LithStatus in bridge.zig -CONSTANT: STATUS_OK 0 -CONSTANT: STATUS_INVALID_ARG 1 -CONSTANT: STATUS_NOT_FOUND 2 -CONSTANT: STATUS_PERMISSION_DENIED 3 -CONSTANT: STATUS_ALREADY_EXISTS 4 -CONSTANT: STATUS_CONSTRAINT_VIOLATION 5 -CONSTANT: STATUS_TYPE_MISMATCH 6 -CONSTANT: STATUS_OUT_OF_MEMORY 7 -CONSTANT: STATUS_IO_ERROR 8 -CONSTANT: STATUS_CORRUPTION 9 -CONSTANT: STATUS_CONFLICT 10 -CONSTANT: STATUS_INTERNAL_ERROR 11 - -! ============================================================ -! Utilities -! ============================================================ - -: factor>json-string ( obj -- str ) - >json utf8 encode >string ; - -: hashtable>json ( hash -- str ) - >json ; - -:: make-c-result ( str status -- result ) - c-string-result malloc-struct - str utf8 string>alien >>data - str length >>length - status >>status ; - -: make-error-result ( status msg -- result ) - swap - H{ - { "status" "error" } - } clone - [ "message" ] dip [ set-at ] keep - hashtable>json - swap make-c-result ; - -! ============================================================ -! C API Functions -! ============================================================ - -! Initialize GQL runtime -:: c_gql_init ( -- status ) - [ - use-memory-storage - STATUS_OK - ] [ - drop STATUS_INTERNAL_ERROR - ] recover ; - -! Initialize with persistent storage path -:: c_gql_init_with_path ( path -- status ) - [ - path utf8 alien>string use-bridge-storage - STATUS_OK - ] [ - drop STATUS_INTERNAL_ERROR - ] recover ; - -! Execute GQL query and return JSON result -:: c_gql_execute ( query_str -- result ) - [ - ! Convert C string to Factor string - query_str utf8 alien>string :> query - - ! Parse and execute - query run-gql :> result-hash - - ! Convert result to JSON - result-hash hashtable>json STATUS_OK make-c-result - ] [ - | err | - ! Error handling - STATUS_INTERNAL_ERROR - err error-summary - make-error-result - ] recover ; - -! Free C result -:: c_gql_free_result ( result -- ) - result data>> [ free ] when* - result free ; - -! Close GQL runtime -:: c_gql_close ( -- status ) - [ - close-backend - STATUS_OK - ] [ - drop STATUS_INTERNAL_ERROR - ] recover ; - -! ============================================================ -! Query Plan API -! ============================================================ - -! Get query plan without executing -:: c_gql_explain ( query_str -- result ) - [ - query_str utf8 alien>string :> query - query explain-gql :> plan-hash - plan-hash hashtable>json STATUS_OK make-c-result - ] [ - | err | - STATUS_INTERNAL_ERROR - err error-summary - make-error-result - ] recover ; - -! Get query plan with execution timing -:: c_gql_explain_analyze ( query_str -- result ) - [ - query_str utf8 alien>string :> query - query explain-analyze-gql :> plan-hash - plan-hash hashtable>json STATUS_OK make-c-result - ] [ - | err | - STATUS_INTERNAL_ERROR - err error-summary - make-error-result - ] recover ; - -! ============================================================ -! Collection Management API -! ============================================================ - -! List all collections -:: c_gql_list_collections ( -- result ) - [ - storage-list-collections >array :> collections - H{ - { "status" "ok" } - { "collections" collections } - } hashtable>json STATUS_OK make-c-result - ] [ - | err | - STATUS_INTERNAL_ERROR - err error-summary - make-error-result - ] recover ; - -! Get collection schema -:: c_gql_get_schema ( collection_name -- result ) - [ - collection_name utf8 alien>string :> coll - coll storage-get-collection :> docs - docs first [ keys >array ] [ { } ] if* :> fields - H{ - { "status" "ok" } - { "collection" coll } - { "fields" fields } - } hashtable>json STATUS_OK make-c-result - ] [ - | err | - STATUS_NOT_FOUND - err error-summary - make-error-result - ] recover ; diff --git a/lithoglyph/core-factor/gql/benchmarks.factor b/lithoglyph/core-factor/gql/benchmarks.factor deleted file mode 100644 index 1ff978b3..00000000 --- a/lithoglyph/core-factor/gql/benchmarks.factor +++ /dev/null @@ -1,322 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Form.Runtime - Performance Benchmarks -! -! Benchmarks for Parser, Planner, Executor, and Normalizer components. -! Used for establishing baseline performance and detecting regressions. - -USING: accessors arrays assocs calendar gql fd-discovery formatting -io kernel locals math math.parser math.statistics namespaces random -sequences system vectors ; - -IN: benchmarks - -! ============================================================ -! Timing Utilities -! ============================================================ - -: measure-ns ( quot -- result nanoseconds ) - nano-count [ call ] dip nano-count swap - ; inline - -: measure-ms ( quot -- result milliseconds ) - measure-ns 1000000.0 / ; inline - -: average-time-ms ( quot n -- avg-ms ) - [ measure-ms nip ] curry replicate mean ; inline - -: benchmark-with-warmup ( quot warmup-runs bench-runs -- avg-ms ) - [ 2dup [ drop call drop ] curry times ] dip - average-time-ms ; inline - -! ============================================================ -! Test Data Generation -! ============================================================ - -: random-string ( len -- str ) - [ CHAR: a CHAR: z [a..b] random ] "" replicate-as ; - -: random-document ( -- doc ) - H{ } clone - 10 random-string "id" pick set-at - 8 random-string "name" pick set-at - { "Engineering" "Sales" "Marketing" "HR" "Finance" } random "dept" pick set-at - 50000 100000 [a..b] random number>string "salary" pick set-at - 1990 2000 [a..b] random number>string "start_year" pick set-at ; - -:: generate-test-data ( n -- data ) - V{ } clone :> data - n [ random-document data push ] times - data ; - -:: setup-benchmark-collection ( name n -- ) - collections get clear-assoc - n generate-test-data name set-collection ; - -! ============================================================ -! Parser Benchmarks -! ============================================================ - -: bench-parse-simple-select ( -- ms ) - [ "SELECT * FROM users" parse-gql drop ] 100 average-time-ms ; - -: bench-parse-complex-select ( -- ms ) - [ "SELECT name, dept, salary FROM employees WHERE dept = Engineering LIMIT 100 OFFSET 10" parse-gql drop ] - 100 average-time-ms ; - -: bench-parse-insert ( -- ms ) - [ "INSERT INTO users { name: Test, dept: HR }" parse-gql drop ] - 100 average-time-ms ; - -: bench-parse-explain ( -- ms ) - [ "EXPLAIN ANALYZE VERBOSE SELECT * FROM users WHERE salary > 50000" parse-gql drop ] - 100 average-time-ms ; - -: run-parser-benchmarks ( -- ) - "=== Parser Benchmarks ===" print - bench-parse-simple-select "Simple SELECT: %.3f ms" sprintf print - bench-parse-complex-select "Complex SELECT: %.3f ms" sprintf print - bench-parse-insert "INSERT: %.3f ms" sprintf print - bench-parse-explain "EXPLAIN ANALYZE: %.3f ms" sprintf print - "" print ; - -! ============================================================ -! Planner Benchmarks -! ============================================================ - -: bench-plan-simple-select ( -- ms ) - "SELECT * FROM users" parse-gql :> ast - [ ast plan-query drop ] 100 average-time-ms ; - -: bench-plan-complex-select ( -- ms ) - "SELECT name, dept FROM users WHERE dept = Engineering LIMIT 50" parse-gql :> ast - [ ast plan-query drop ] 100 average-time-ms ; - -: bench-plan-insert ( -- ms ) - "INSERT INTO users { name: Test }" parse-gql :> ast - [ ast plan-query drop ] 100 average-time-ms ; - -: run-planner-benchmarks ( -- ) - "=== Planner Benchmarks ===" print - bench-plan-simple-select "Simple SELECT plan: %.3f ms" sprintf print - bench-plan-complex-select "Complex SELECT plan: %.3f ms" sprintf print - bench-plan-insert "INSERT plan: %.3f ms" sprintf print - "" print ; - -! ============================================================ -! Executor Benchmarks -! ============================================================ - -:: bench-executor-select ( n -- ms ) - "bench_select" n setup-benchmark-collection - [ "SELECT * FROM bench_select" run-gql drop ] - 10 average-time-ms ; - -:: bench-executor-select-filtered ( n -- ms ) - "bench_filter" n setup-benchmark-collection - [ "SELECT * FROM bench_filter WHERE dept = Engineering" run-gql drop ] - 10 average-time-ms ; - -:: bench-executor-insert ( n -- ms ) - "bench_insert" 0 setup-benchmark-collection - [ - n [ - random-document :> doc - "INSERT INTO bench_insert { name: X }" run-gql drop - ] times - ] measure-ms nip ; - -:: bench-executor-update ( n -- ms ) - "bench_update" n setup-benchmark-collection - [ "UPDATE bench_update SET salary = 999999 WHERE dept = Engineering" run-gql drop ] - 10 average-time-ms ; - -: run-executor-benchmarks ( -- ) - "=== Executor Benchmarks ===" print - - "SELECT (100 docs):" print - 100 bench-executor-select " Full scan: %.3f ms" sprintf print - 100 bench-executor-select-filtered " Filtered: %.3f ms" sprintf print - - "SELECT (1000 docs):" print - 1000 bench-executor-select " Full scan: %.3f ms" sprintf print - 1000 bench-executor-select-filtered " Filtered: %.3f ms" sprintf print - - "SELECT (10000 docs):" print - 10000 bench-executor-select " Full scan: %.3f ms" sprintf print - 10000 bench-executor-select-filtered " Filtered: %.3f ms" sprintf print - - "INSERT (100 docs):" print - 100 bench-executor-insert " Batch: %.3f ms" sprintf print - - "UPDATE (1000 docs):" print - 1000 bench-executor-update " Filtered update: %.3f ms" sprintf print - - "" print ; - -! ============================================================ -! FD Discovery Benchmarks -! ============================================================ - -:: bench-fd-discovery ( n -- ms ) - "bench_fd" n setup-benchmark-collection - "bench_fd" get-collection :> data - [ - data - fd-discovery-config new - 1000 >>sample-size - 0.95 >>confidence-threshold - "dfd" >>algorithm - 3 >>max-lhs-size - run-dfd drop - ] measure-ms nip ; - -:: bench-fd-discovery-full ( n -- ms ) - "bench_fd_full" n setup-benchmark-collection - "bench_fd_full" get-collection :> data - [ - data - fd-discovery-config new - n >>sample-size ! Full dataset - 0.95 >>confidence-threshold - "dfd" >>algorithm - 4 >>max-lhs-size ! Deeper search - run-dfd drop - ] measure-ms nip ; - -: run-fd-discovery-benchmarks ( -- ) - "=== FD Discovery Benchmarks ===" print - - "DFD (sampled, max-lhs=3):" print - 100 bench-fd-discovery " 100 docs: %.3f ms" sprintf print - 500 bench-fd-discovery " 500 docs: %.3f ms" sprintf print - 1000 bench-fd-discovery " 1000 docs: %.3f ms" sprintf print - - "DFD (full, max-lhs=4):" print - 100 bench-fd-discovery-full " 100 docs: %.3f ms" sprintf print - 200 bench-fd-discovery-full " 200 docs: %.3f ms" sprintf print - - "" print ; - -! ============================================================ -! Normal Form Analysis Benchmarks -! ============================================================ - -:: bench-nf-analysis ( num-fds -- ms ) - ! Generate synthetic FDs - V{ } clone :> fds - num-fds [ - functional-dependency new - 1array "attr" swap number>string append >>determinant - { "dependent" } >>dependent - 1.0 >>confidence - fds push - ] each-integer - - { { "id" } } :> keys ! Simple key - - [ fds keys analyze-normal-form drop ] - 100 average-time-ms ; - -: run-nf-analysis-benchmarks ( -- ) - "=== Normal Form Analysis Benchmarks ===" print - 5 bench-nf-analysis "5 FDs: %.3f ms" sprintf print - 10 bench-nf-analysis "10 FDs: %.3f ms" sprintf print - 20 bench-nf-analysis "20 FDs: %.3f ms" sprintf print - 50 bench-nf-analysis "50 FDs: %.3f ms" sprintf print - "" print ; - -! ============================================================ -! End-to-End Pipeline Benchmarks -! ============================================================ - -:: bench-full-pipeline ( n -- ms ) - "bench_pipeline" n setup-benchmark-collection - [ - ! Parse - "SELECT * FROM bench_pipeline WHERE dept = Engineering" parse-gql :> ast - ! Plan - ast plan-query :> plan - ! Execute - ast execute-gql :> result - ! Discover FDs - "bench_pipeline" get-collection - fd-discovery-config new - 100 >>sample-size - 0.95 >>confidence-threshold - "dfd" >>algorithm - 3 >>max-lhs-size - run-dfd :> fds - ! Analyze normal form - fds dependencies>> { { "id" } } analyze-normal-form drop - ] measure-ms nip ; - -: run-pipeline-benchmarks ( -- ) - "=== Full Pipeline Benchmarks ===" print - 100 bench-full-pipeline "100 docs: %.3f ms" sprintf print - 500 bench-full-pipeline "500 docs: %.3f ms" sprintf print - 1000 bench-full-pipeline "1000 docs: %.3f ms" sprintf print - "" print ; - -! ============================================================ -! Memory Usage Estimation -! ============================================================ - -:: estimate-collection-memory ( n -- bytes ) - ! Rough estimate: each document ~200 bytes - n 200 * ; - -: run-memory-estimates ( -- ) - "=== Memory Estimates ===" print - 100 estimate-collection-memory "100 docs: ~%d bytes" sprintf print - 1000 estimate-collection-memory "1000 docs: ~%d bytes" sprintf print - 10000 estimate-collection-memory "10000 docs: ~%d bytes" sprintf print - 100000 estimate-collection-memory "100000 docs: ~%d bytes (~%d MB)" [ 1048576 / ] keep 2array vsprintf print - "" print ; - -! ============================================================ -! Benchmark Report Generation -! ============================================================ - -: generate-benchmark-report ( -- ) - "Lith Performance Benchmark Report" print - "=" 50 concat print - now timestamp>rfc3339 "Generated: %s" sprintf print - "" print - - run-parser-benchmarks - run-planner-benchmarks - run-executor-benchmarks - run-fd-discovery-benchmarks - run-nf-analysis-benchmarks - run-pipeline-benchmarks - run-memory-estimates - - "=" 50 concat print - "Benchmark complete." print ; - -! ============================================================ -! Quick Benchmark (for CI) -! ============================================================ - -: quick-benchmark ( -- success? ) - ! Run abbreviated benchmarks, return false if any exceed thresholds - t :> passed! - - ! Parser should be < 1ms - bench-parse-simple-select 1.0 > [ f passed! ] when - - ! SELECT 100 docs should be < 10ms - 100 bench-executor-select 10.0 > [ f passed! ] when - - ! FD discovery on 100 docs should be < 100ms - 100 bench-fd-discovery 100.0 > [ f passed! ] when - - passed ; - -! ============================================================ -! Main Entry Point -! ============================================================ - -: run-benchmarks ( -- ) - generate-benchmark-report ; - -MAIN: run-benchmarks diff --git a/lithoglyph/core-factor/gql/gql.factor b/lithoglyph/core-factor/gql/gql.factor deleted file mode 100644 index 8e24cd5c..00000000 --- a/lithoglyph/core-factor/gql/gql.factor +++ /dev/null @@ -1,896 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Form.Runtime - GQL Parser and Executor -! -! GQL (Glyph Query Language) - Carving queries into stone-carved data. -! Part of Lithoglyph: Stone-carved data for the ages. - -USING: accessors arrays assocs combinators combinators.short-circuit -continuations formatting generalizations io json kernel locals math -math.parser peg peg.ebnf random sequences splitting storage-backend -strings system unicode vectors ; - -IN: gql - -! ============================================================ -! AST Node Types -! ============================================================ - -TUPLE: gql-insert collection document provenance ; -TUPLE: gql-select fields collection where-clause edge-clause limit-clause with-provenance? ; -TUPLE: gql-update collection assignments where-clause provenance ; -TUPLE: gql-delete collection where-clause provenance ; -TUPLE: gql-create collection fields schema ; -TUPLE: gql-drop collection provenance ; -TUPLE: gql-explain inner-stmt analyze? verbose? ; -TUPLE: gql-introspect target arg ; - -TUPLE: edge-clause type direction depth where ; -TUPLE: where-clause expression ; -TUPLE: limit-clause limit offset ; - -TUPLE: comparison field op value ; -TUPLE: binary-expr left op right ; - -! ============================================================ -! Tokenizer -! ============================================================ - -: skip-whitespace ( str -- str' ) - [ " \t\n\r" member? not ] find drop "" or ; - -: keyword? ( str -- ? ) - >upper { - "SELECT" "FROM" "WHERE" "INSERT" "INTO" "UPDATE" - "DELETE" "SET" "CREATE" "DROP" "COLLECTION" - "WITH" "PROVENANCE" "LIMIT" "OFFSET" - "TRAVERSE" "OUTBOUND" "INBOUND" "ANY" "DEPTH" - "EXPLAIN" "INTROSPECT" "SCHEMA" "CONSTRAINTS" - "JOURNAL" "SINCE" "COLLECTIONS" - "AND" "OR" "NOT" "NULL" "TRUE" "FALSE" - "STRING" "INTEGER" "FLOAT" "BOOLEAN" "TIMESTAMP" - "JSON" "PROMPT_SCORE" "UNIQUE" "CHECK" "REFERENCES" - "LIKE" "IN" "CONTAINS" - } member? ; - -: split-tokens ( str -- tokens ) - ! Simple tokenizer - splits on whitespace and punctuation - " \t\n\r" split harvest - [ "," = not ] filter ; - -! ============================================================ -! Parser Combinators (Simplified) -! ============================================================ - -ERROR: gql-parse-error message position ; - -: peek-token ( tokens -- token/f ) - dup empty? [ drop f ] [ first ] if ; - -: consume-token ( tokens -- tokens' token ) - unclip-slice ; ! Returns ( rest first ) = ( tokens' token ), token on top - -: expect-token ( tokens expected -- tokens' ) - over peek-token ! ( tokens expected token ) - swap >upper swap >upper ! ( tokens EXPECTED TOKEN ) - over = [ ! ( tokens EXPECTED ) if match - drop consume-token drop ! ( tokens' ) - ] [ - "Expected '" "'" surround gql-parse-error - ] if ; - -: try-consume ( tokens expected -- tokens' matched? ) - over peek-token ! ( tokens expected actual/f ) - dup [ - ! Have a token - check if it matches - swap >upper swap >upper ! ( tokens EXPECTED ACTUAL ) - over over = ! ( tokens EXPECTED ACTUAL match? ) - [ - 2drop ! ( tokens ) - unclip-slice drop ! ( tokens' ) - t ! ( tokens' t ) - ] [ - 2drop ! ( tokens ) - f ! ( tokens f ) - ] if - ] [ - ! No token - drop drop f ! ( tokens f ) - ] if ; - -! ============================================================ -! Parse Primitives -! ============================================================ - -: parse-identifier ( tokens -- tokens' identifier ) - consume-token - dup keyword? [ "Unexpected keyword" gql-parse-error ] when ; - -: parse-json-value ( tokens -- tokens' value ) - ! Simplified: just consume until end of JSON - consume-token - dup "{" = [ - drop - ! Parse JSON object - H{ } clone - [ over peek-token "}" = not ] [ - swap consume-token drop ! key - swap consume-token drop ! : - swap consume-token ! value - ! Would need full JSON parsing here - 2drop - ] while - swap consume-token drop ! } - swap - ] [ - ! It's a simple value - swap - ] if ; - -: parse-string-literal ( tokens -- tokens' string ) - consume-token - ! Remove quotes if present - dup first CHAR: " = [ - 1 tail* dup length 1 - head* - ] when ; - -! ============================================================ -! Parse Statements -! ============================================================ - -: parse-collection-name ( tokens -- tokens' name ) - parse-identifier ; - -: parse-field-list ( tokens -- tokens' fields ) - ! Collect field names until we see "FROM" - V{ } clone swap ! ( accum tokens ) - [ dup peek-token "FROM" = not ] [ - consume-token ! ( accum tokens' field ) - [ swap ] dip ! ( tokens' accum field ) - suffix ! ( tokens' accum' ) - swap ! ( accum' tokens' ) - dup peek-token "," = [ consume-token drop ] when - ] while - swap >array ; ! ( tokens' fields ) - -: parse-where-clause ( tokens -- tokens' where/f ) - "WHERE" try-consume [ - ! Parse expression: field op value - parse-identifier ! ( tokens' field ) - [ consume-token ] dip ! ( tokens'' op field ) - [ consume-token ] dip ! ( tokens''' value op field ) - swap rot ! ( tokens''' field op value ) - comparison boa ! ( tokens''' comparison ) - where-clause boa ! ( tokens''' where ) - ] [ - f - ] if ; - -:: parse-edge-clause ( tokens -- tokens' edge/f ) - tokens "TRAVERSE" try-consume :> ( tokens' matched? ) - matched? [ - tokens' parse-identifier :> ( tokens'' edge-type ) - tokens'' consume-token :> ( tokens''' dir-raw ) - dir-raw >upper :> direction - tokens''' "DEPTH" try-consume :> ( tokens'''' has-depth? ) - has-depth? [ - tokens'''' consume-token string>number :> ( tokens''''' depth ) - tokens''''' - edge-type direction depth f edge-clause boa - ] [ - tokens'''' - edge-type direction 1 f edge-clause boa - ] if - ] [ - tokens f - ] if ; - -:: parse-limit-clause ( tokens -- tokens' limit/f ) - tokens "LIMIT" try-consume :> ( tokens' matched? ) - matched? [ - tokens' consume-token string>number :> ( tokens'' limit ) - tokens'' "OFFSET" try-consume :> ( tokens''' has-offset? ) - has-offset? [ - tokens''' consume-token string>number :> ( tokens'''' offset ) - tokens'''' - limit offset limit-clause boa - ] [ - tokens''' - limit 0 limit-clause boa - ] if - ] [ - tokens f - ] if ; - -: parse-provenance-clause ( tokens -- tokens' prov/f ) - "WITH" try-consume [ - "PROVENANCE" expect-token - ! Parse JSON object - consume-token drop ! { - H{ } clone ! placeholder - [ over peek-token "}" = not ] [ - swap consume-token drop ! skip tokens until } - ] while - swap consume-token drop ! } - swap - ] [ - f - ] if ; - -! ============================================================ -! Statement Parsers -! ============================================================ - -:: parse-insert ( tokens -- tokens' ast ) - tokens "INTO" expect-token parse-collection-name :> ( tokens' collection ) - ! Parse document body (simplified - just consume JSON) - tokens' consume-token drop :> tokens'' ! consume { - H{ } clone :> doc - tokens'' [ dup peek-token "}" = not ] [ - consume-token drop - ] while :> tokens''' - tokens''' consume-token drop :> tokens'''' ! consume } - ! Parse provenance - tokens'''' parse-provenance-clause :> ( final-tokens prov ) - final-tokens - collection doc prov gql-insert boa ; - -: parse-select ( tokens -- tokens' ast ) - ! Parse "SELECT fields FROM collection" (simplified) - ! Stack discipline: always keep tokens on top until the end - - ! Parse field list - dup peek-token "*" = [ - consume-token drop { "*" } ! ( tokens' { "*" } ) - ] [ - parse-field-list ! ( tokens' fields ) - ] if - ! Stack: ( tokens' fields ) - - ! FROM collection - swap ! ( fields tokens' ) - "FROM" expect-token ! ( fields tokens'' ) - parse-collection-name ! ( fields tokens'' collection ) - - ! Build the gql-select tuple with defaults for optional clauses - ! gql-select needs: ( fields collection where edge lim prov ) - ! Stack has: ( fields tokens'' collection ) - - rot ! ( tokens'' collection fields ) - swap ! ( tokens'' fields collection ) - f f f f ! ( tokens'' fields collection f f f f ) - gql-select boa ! ( tokens'' ast ) - ; - -:: parse-update ( tokens -- tokens' ast ) - tokens parse-collection-name :> ( tokens' collection ) - tokens' "SET" expect-token :> tokens'' - ! Parse assignments (simplified) - V{ } clone :> assignments - tokens'' [ dup peek-token "WHERE" = not ] [ - parse-identifier :> ( t field ) - t consume-token drop :> t' ! = - t' consume-token :> ( t'' value ) - field value 2array assignments push - t'' dup peek-token "," = [ consume-token drop ] when - ] while :> tokens''' - tokens''' parse-where-clause :> ( tokens'''' where ) - tokens'''' parse-provenance-clause :> ( final-tokens prov ) - final-tokens - collection assignments >array where prov gql-update boa ; - -:: parse-delete ( tokens -- tokens' ast ) - tokens "FROM" expect-token parse-collection-name :> ( tokens' collection ) - tokens' parse-where-clause :> ( tokens'' where ) - tokens'' parse-provenance-clause :> ( final-tokens prov ) - final-tokens - collection where prov gql-delete boa ; - -:: parse-create ( tokens -- tokens' ast ) - tokens "COLLECTION" expect-token parse-collection-name :> ( tokens' collection ) - ! Parse optional field definitions - tokens' peek-token "(" = [ - tokens' consume-token drop :> tokens'' ! consume ( - V{ } clone :> field-defs - tokens'' [ dup peek-token ")" = not ] [ - parse-identifier :> ( t field ) - t consume-token :> ( t' type ) - field type 2array field-defs push - t' dup peek-token "," = [ consume-token drop ] when - ] while :> tokens''' - tokens''' consume-token drop :> tokens'''' ! consume ) - tokens'''' field-defs >array - ] [ - tokens' { } - ] if :> ( tokens''''' fields ) - ! Parse optional schema - tokens''''' "WITH" try-consume :> ( tokens'''''' has-schema? ) - has-schema? [ - tokens'''''' "SCHEMA" expect-token :> tokens''''''' - tokens''''''' - collection fields H{ } clone gql-create boa ! placeholder schema - ] [ - tokens'''''' - collection fields f gql-create boa - ] if ; - -:: parse-drop ( tokens -- tokens' ast ) - tokens "COLLECTION" expect-token parse-collection-name :> ( tokens' collection ) - tokens' parse-provenance-clause :> ( final-tokens prov ) - final-tokens - collection prov gql-drop boa ; - -:: parse-introspect-target ( tokens -- tokens' target arg ) - tokens consume-token :> ( tokens' target-raw ) - target-raw >upper :> target - target "JOURNAL" = [ - tokens' "SINCE" try-consume :> ( tokens'' has-since? ) - has-since? [ - tokens'' consume-token string>number :> ( tokens''' since-val ) - tokens''' target since-val - ] [ - tokens'' target 0 - ] if - ] [ - target "SCHEMA" = target "CONSTRAINTS" = or [ - tokens' peek-token :> next-tok - next-tok keyword? not next-tok and [ - tokens' parse-identifier :> ( tokens'' arg ) - tokens'' target arg - ] [ - tokens' target f - ] if - ] [ - tokens' target f - ] if - ] if ; - -: parse-introspect ( tokens -- tokens' ast ) - parse-introspect-target - gql-introspect boa ; - -DEFER: parse-statement - -:: parse-explain ( tokens -- tokens' ast ) - ! Parse optional ANALYZE and VERBOSE flags - f :> analyze?! - f :> verbose?! - tokens - "ANALYZE" try-consume [ t analyze?! ] when - "VERBOSE" try-consume [ t verbose?! ] when - parse-statement :> ( tokens' inner ) - tokens' - inner analyze? verbose? gql-explain boa ; - -: parse-statement ( tokens -- tokens' ast ) - consume-token >upper { ! After consume-token: ( tokens' token ), >upper: ( tokens' TOKEN ) - { "INSERT" [ parse-insert ] } - { "SELECT" [ parse-select ] } - { "UPDATE" [ parse-update ] } - { "DELETE" [ parse-delete ] } - { "CREATE" [ parse-create ] } - { "DROP" [ parse-drop ] } - { "EXPLAIN" [ parse-explain ] } - { "INTROSPECT" [ parse-introspect ] } - [ "Unknown statement type" gql-parse-error ] - } case ; - -! ============================================================ -! Main Parser Entry Point -! ============================================================ - -: parse-gql ( str -- ast ) - ! Remove trailing semicolon if present - dup ";" tail? [ but-last ] when - ! Remove comments - "\n" split - [ - "--" split1 drop ! Remove line comments - "" or - ] map - " " join - ! Tokenize - split-tokens - ! Parse - parse-statement - ! Should have consumed all tokens - nip ; - -! ============================================================ -! Query Plan Types -! ============================================================ - -TUPLE: plan-step - type ! scan | index-lookup | filter | project | sort | limit | join - target ! collection name or subplan - cost ! estimated cost - rows ! estimated rows - rationale ; ! why this step was chosen - -TUPLE: query-plan - steps ! sequence of plan-steps - total-cost ! sum of costs - provenance? ; ! whether provenance is requested - -! ============================================================ -! Query Planner -! ============================================================ - -GENERIC: plan-query ( ast -- plan ) - -:: make-scan-step ( collection where -- step ) - plan-step new - "scan" >>type - collection >>target - where [ 100 ] [ 1000 ] if >>cost ! filter reduces cost estimate - where [ 10 ] [ 100 ] if >>rows - where [ - "Full collection scan with filter - consider adding index" - ] [ - "Full collection scan - no filter specified" - ] if >>rationale ; - -:: make-project-step ( fields -- step ) - plan-step new - "project" >>type - fields >>target - 1 >>cost - 0 >>rows - fields { "*" } = [ - "Selecting all fields" - ] [ - fields length "Projecting %d field(s)" sprintf - ] if >>rationale ; - -:: make-limit-step ( lim -- step ) - plan-step new - "limit" >>type - lim >>target - 1 >>cost - lim limit>> >>rows - lim limit>> "Limiting to %d rows" sprintf >>rationale ; - -:: make-edge-step ( edge -- step ) - plan-step new - "traverse" >>type - edge type>> >>target - edge depth>> 10 * >>cost - edge depth>> 5 * >>rows - edge direction>> edge depth>> "Traversing %s edges to depth %d" sprintf >>rationale ; - -M: gql-select plan-query - query-plan new - V{ } clone - ! Add project step - over fields>> make-project-step suffix - ! Add scan step - over collection>> over where-clause>> make-scan-step suffix - ! Add edge traversal if present - over edge-clause>> [ make-edge-step suffix ] when* - ! Add limit if present - over limit-clause>> [ make-limit-step suffix ] when* - >>steps - ! Calculate total cost - dup steps>> [ cost>> ] map-sum >>total-cost - ! Check provenance flag - swap with-provenance?>> >>provenance? ; - -M: gql-insert plan-query - query-plan new - V{ } - over collection>> - plan-step new - "insert" >>type - swap >>target - 10 >>cost - 1 >>rows - "Insert document into collection" >>rationale - suffix >>steps - 10 >>total-cost - f >>provenance? ; - -M: gql-update plan-query - query-plan new - V{ } - over collection>> over where-clause>> make-scan-step suffix - over collection>> - plan-step new - "update" >>type - swap >>target - 50 >>cost - 0 >>rows - "Update matching documents" >>rationale - suffix >>steps - 60 >>total-cost - f >>provenance? ; - -M: gql-delete plan-query - query-plan new - V{ } - over collection>> over where-clause>> make-scan-step suffix - over collection>> - plan-step new - "delete" >>type - swap >>target - 50 >>cost - 0 >>rows - "Delete matching documents" >>rationale - suffix >>steps - 60 >>total-cost - f >>provenance? ; - -M: gql-create plan-query - query-plan new - V{ } - over collection>> - plan-step new - "create-collection" >>type - swap >>target - 5 >>cost - 0 >>rows - "Create new collection with schema" >>rationale - suffix >>steps - 5 >>total-cost - f >>provenance? ; - -M: gql-drop plan-query - query-plan new - V{ } - over collection>> - plan-step new - "drop-collection" >>type - swap >>target - 5 >>cost - 0 >>rows - "Drop collection and all documents" >>rationale - suffix >>steps - 5 >>total-cost - f >>provenance? ; - -M: gql-explain plan-query - inner-stmt>> plan-query ; - -M: gql-introspect plan-query - query-plan new - V{ } - over target>> - plan-step new - "introspect" >>type - swap >>target - 1 >>cost - 0 >>rows - "Read system metadata" >>rationale - suffix >>steps - 1 >>total-cost - f >>provenance? ; - -! ============================================================ -! Plan Rendering (for EXPLAIN) -! ============================================================ - -: step>assoc ( step -- assoc ) - { - [ type>> "type" swap 2array ] - [ target>> "target" swap 2array ] - [ cost>> "cost" swap 2array ] - [ rows>> "estimated_rows" swap 2array ] - [ rationale>> "rationale" swap 2array ] - } cleave 5 narray >hashtable ; - -: plan>assoc ( plan -- assoc ) - { - [ steps>> [ step>assoc ] map "steps" swap 2array ] - [ total-cost>> "total_cost" swap 2array ] - [ provenance?>> "with_provenance" swap 2array ] - } cleave 3 narray >hashtable ; - -! ============================================================ -! Query Executor -! ============================================================ - -GENERIC: execute-gql ( ast -- result ) - -! Storage backend - uses pluggable storage (memory or bridge) -! See storage-backend.factor for backend implementations - -: get-collection ( name -- docs ) - storage-get-collection ; - -: set-collection ( docs name -- ) - storage-set-collection ; - -: generate-doc-id ( -- id ) - 32 [ CHAR: a CHAR: z [a..b] random ] "" replicate-as ; - -:: execute-insert ( collection doc prov -- result ) - collection get-collection :> coll - generate-doc-id :> doc-id - doc-id "id" doc clone [ set-at ] keep :> doc' - doc' coll push - coll collection set-collection - H{ - { "status" "ok" } - { "document_id" doc-id } - { "collection" collection } - } clone - prov [ "provenance" swap pick set-at ] when* ; - -:: match-where? ( doc where -- ? ) - where [ - where expression>> :> comp - comp field>> doc at :> actual - comp value>> :> expected - comp op>> { - { "=" [ actual expected = ] } - { "!=" [ actual expected = not ] } - { ">" [ actual expected > ] } - { "<" [ actual expected < ] } - { ">=" [ actual expected >= ] } - { "<=" [ actual expected <= ] } - { "LIKE" [ actual expected swap subseq? ] } - { "CONTAINS" [ actual expected swap member? ] } - [ 3drop f ] - } case - ] [ t ] if ; - -:: execute-select ( fields collection where edge lim prov? -- result ) - collection get-collection :> all-docs - ! Apply filter - all-docs [ where match-where? ] filter :> filtered - ! Apply limit - lim [ [ limit>> head-clamp ] keep offset>> tail-clamp ] when* :> limited - ! Project fields - fields { "*" } = [ - limited - ] [ - limited [ [ fields ] dip '[ _ swap at ] map>alist >hashtable ] map - ] if :> projected - H{ - { "status" "ok" } - { "collection" collection } - { "count" projected length } - { "rows" projected >array } - } clone - prov? [ "provenance_enabled" t pick set-at ] when ; - -:: execute-update ( collection assignments where prov -- result ) - collection get-collection :> docs - 0 :> modified! - docs [ - dup where match-where? [ - assignments [ first2 pick set-at ] each - modified 1 + modified! - ] when - ] each - docs collection set-collection - H{ - { "status" "ok" } - { "collection" collection } - { "modified_count" modified } - } clone - prov [ "provenance" swap pick set-at ] when* ; - -:: execute-delete ( collection where prov -- result ) - collection get-collection :> docs - docs [ where match-where? not ] filter :> remaining - docs length remaining length - :> deleted - remaining collection set-collection - H{ - { "status" "ok" } - { "collection" collection } - { "deleted_count" deleted } - } clone - prov [ "provenance" swap pick set-at ] when* ; - -:: execute-create ( collection fields schema -- result ) - V{ } clone collection set-collection - H{ - { "status" "ok" } - { "collection" collection } - { "schema_version" 1 } - { "fields" fields } - } clone ; - -:: execute-drop ( collection prov -- result ) - f collection set-collection - H{ - { "status" "ok" } - { "collection" collection } - { "dropped" t } - } clone - prov [ "provenance" swap pick set-at ] when* ; - -M: gql-insert execute-gql - [ collection>> ] [ document>> ] [ provenance>> ] tri - execute-insert ; - -M: gql-select execute-gql - [ fields>> ] - [ collection>> ] - [ where-clause>> ] - [ edge-clause>> ] - [ limit-clause>> ] - [ with-provenance?>> ] 6 ncleave - execute-select ; - -M: gql-update execute-gql - [ collection>> ] [ assignments>> ] [ where-clause>> ] [ provenance>> ] quad - execute-update ; - -M: gql-delete execute-gql - [ collection>> ] [ where-clause>> ] [ provenance>> ] tri - execute-delete ; - -M: gql-create execute-gql - [ collection>> ] [ fields>> ] [ schema>> ] tri - execute-create ; - -M: gql-drop execute-gql - [ collection>> ] [ provenance>> ] bi - execute-drop ; - -! Timing helper for ANALYZE mode -: with-timing ( quot -- result elapsed-ms ) - nano-count [ call ] dip nano-count swap - 1000000 / ; inline - -! Generate verbose plan description -:: plan-step>verbose ( step -- string ) - step type>> :> type - step target>> :> target - step cost>> :> cost - step rows>> :> rows - step rationale>> :> rationale - { - { [ type "scan" = ] [ - target "-> Seq Scan on %s" sprintf - "\n Estimated Cost: " cost number>string append - "\n Estimated Rows: " rows number>string append - "\n Note: " rationale append - append append append - ] } - { [ type "project" = ] [ - "-> Projection" - target array? [ - "\n Columns: " target ", " join append - ] when - append - ] } - { [ type "limit" = ] [ - "-> Limit" - target limit-clause? [ - "\n Limit: " target limit>> number>string append - target offset>> 0 > [ - "\n Offset: " target offset>> number>string append - ] when - ] when - append - ] } - { [ type "traverse" = ] [ - target "-> Graph Traversal (%s)" sprintf - "\n Estimated Cost: " cost number>string append - append - ] } - { [ type "insert" = ] [ - target "-> Insert into %s" sprintf - ] } - { [ type "update" = ] [ - target "-> Update on %s" sprintf - ] } - { [ type "delete" = ] [ - target "-> Delete from %s" sprintf - ] } - { [ type "create-collection" = ] [ - target "-> Create Collection %s" sprintf - ] } - { [ type "drop-collection" = ] [ - target "-> Drop Collection %s" sprintf - ] } - { [ type "introspect" = ] [ - target "-> Introspect %s" sprintf - ] } - [ drop type "-> %s" sprintf ] - } cond ; - -:: plan>verbose-string ( plan -- string ) - "QUERY PLAN\n" :> out! - "-" 50 concat "\n" append out swap append out! - plan steps>> [| step i | - " " i 2 * concat - step plan-step>verbose append - "\n" append - out swap append out! - ] each-index - "\nTotal Cost: " plan total-cost>> number>string append "\n" append - out swap append out! - plan provenance?>> [ "With Provenance Tracking\n" out swap append out! ] when - out ; - -:: execute-explain ( stmt -- result ) - stmt inner-stmt>> plan-query :> plan - stmt analyze?>> [ - ! ANALYZE mode: actually run the query and report timing - [ stmt inner-stmt>> execute-gql ] with-timing :> ( result elapsed ) - H{ - { "status" "ok" } - } clone - "plan" plan plan>assoc pick set-at - "execution_time_ms" elapsed pick set-at - "actual_result" result pick set-at - stmt verbose?>> [ - "verbose_plan" plan plan>verbose-string pick set-at - ] when - ] [ - ! Plain EXPLAIN: just show the plan - H{ - { "status" "ok" } - } clone - "plan" plan plan>assoc pick set-at - stmt verbose?>> [ - "verbose_plan" plan plan>verbose-string pick set-at - ] when - ] if ; - -M: gql-explain execute-gql - execute-explain ; - -M: gql-introspect execute-gql - [ target>> ] [ arg>> ] bi - { - { "SCHEMA" [ - drop ! arg - storage-list-collections - [ get-collection first [ keys ] [ { } ] if* ] map flatten members - H{ - { "status" "ok" } - } clone - [ "fields" ] dip [ set-at ] keep - ] } - { "CONSTRAINTS" [ - drop - H{ - { "status" "ok" } - { "constraints" { } } - { "functional_dependencies" { } } - } - ] } - { "COLLECTIONS" [ - drop - storage-list-collections >array - H{ - { "status" "ok" } - } clone - [ "collections" ] dip [ set-at ] keep - ] } - { "JOURNAL" [ - ! arg is since sequence number - H{ - { "status" "ok" } - { "entries" { } } - { "head" 0 } - } clone - [ "since" ] dip [ set-at ] keep - ] } - [ 2drop H{ { "status" "error" } { "message" "Unknown introspect target" } } ] - } case ; - -! ============================================================ -! Public API -! ============================================================ - -: run-gql ( str -- result ) - parse-gql execute-gql ; - -: explain-gql ( str -- plan ) - "EXPLAIN " prepend - run-gql ; - -: explain-verbose-gql ( str -- plan ) - "EXPLAIN VERBOSE " prepend - run-gql ; - -: explain-analyze-gql ( str -- plan ) - "EXPLAIN ANALYZE " prepend - run-gql ; - -: explain-analyze-verbose-gql ( str -- plan ) - "EXPLAIN ANALYZE VERBOSE " prepend - run-gql ; diff --git a/lithoglyph/core-factor/gql/lexer-tests.factor b/lithoglyph/core-factor/gql/lexer-tests.factor deleted file mode 100644 index f1216e11..00000000 --- a/lithoglyph/core-factor/gql/lexer-tests.factor +++ /dev/null @@ -1,480 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Form.Runtime - GQL Lexer Unit Tests -! -! Dedicated tests for the GQL tokenizer and parser primitives: -! keyword recognition, token splitting, whitespace handling, -! identifier parsing, operator handling, and error recovery. -! -! These tests complement seam-tests.factor which tests the full -! pipeline. Here we focus on lexical analysis correctness. - -USING: accessors arrays assocs continuations gql io kernel math -sequences splitting strings tools.test vectors ; - -IN: lexer-tests - -! ============================================================ -! Tokenizer Tests: split-tokens -! ============================================================ - -! Test basic token splitting -{ { "SELECT" "*" "FROM" "users" } } [ - "SELECT * FROM users" split-tokens >array -] unit-test - -{ { "INSERT" "INTO" "docs" } } [ - "INSERT INTO docs" split-tokens >array -] unit-test - -! Test tab and newline splitting -{ { "SELECT" "*" } } [ - "SELECT\t*" split-tokens >array -] unit-test - -{ { "SELECT" "*" "FROM" "t" } } [ - "SELECT\n*\nFROM\nt" split-tokens >array -] unit-test - -! Test comma filtering -{ { "SELECT" "a" "b" "FROM" "t" } } [ - "SELECT a,b FROM t" split-tokens >array -] unit-test - -{ { "a" "b" "c" } } [ - "a,b,c" split-tokens >array -] unit-test - -! Test multiple whitespace -{ { "SELECT" "*" } } [ - " SELECT * " split-tokens >array -] unit-test - -! Test empty input -{ { } } [ - "" split-tokens >array -] unit-test - -! Test whitespace-only input -{ { } } [ - " " split-tokens >array -] unit-test - -! ============================================================ -! Keyword Recognition Tests: keyword? -! ============================================================ - -! SQL keywords (case-insensitive via >upper) -{ t } [ "SELECT" keyword? ] unit-test -{ t } [ "select" keyword? ] unit-test -{ t } [ "SeLeCt" keyword? ] unit-test -{ t } [ "FROM" keyword? ] unit-test -{ t } [ "from" keyword? ] unit-test -{ t } [ "WHERE" keyword? ] unit-test -{ t } [ "INSERT" keyword? ] unit-test -{ t } [ "INTO" keyword? ] unit-test -{ t } [ "UPDATE" keyword? ] unit-test -{ t } [ "DELETE" keyword? ] unit-test -{ t } [ "SET" keyword? ] unit-test -{ t } [ "CREATE" keyword? ] unit-test -{ t } [ "DROP" keyword? ] unit-test -{ t } [ "COLLECTION" keyword? ] unit-test -{ t } [ "WITH" keyword? ] unit-test -{ t } [ "PROVENANCE" keyword? ] unit-test -{ t } [ "LIMIT" keyword? ] unit-test -{ t } [ "OFFSET" keyword? ] unit-test - -! Graph traversal keywords -{ t } [ "TRAVERSE" keyword? ] unit-test -{ t } [ "OUTBOUND" keyword? ] unit-test -{ t } [ "INBOUND" keyword? ] unit-test -{ t } [ "ANY" keyword? ] unit-test -{ t } [ "DEPTH" keyword? ] unit-test - -! Introspection keywords -{ t } [ "EXPLAIN" keyword? ] unit-test -{ t } [ "INTROSPECT" keyword? ] unit-test -{ t } [ "SCHEMA" keyword? ] unit-test -{ t } [ "CONSTRAINTS" keyword? ] unit-test -{ t } [ "JOURNAL" keyword? ] unit-test -{ t } [ "SINCE" keyword? ] unit-test -{ t } [ "COLLECTIONS" keyword? ] unit-test - -! Boolean and null keywords -{ t } [ "AND" keyword? ] unit-test -{ t } [ "OR" keyword? ] unit-test -{ t } [ "NOT" keyword? ] unit-test -{ t } [ "NULL" keyword? ] unit-test -{ t } [ "TRUE" keyword? ] unit-test -{ t } [ "FALSE" keyword? ] unit-test - -! Type keywords -{ t } [ "STRING" keyword? ] unit-test -{ t } [ "INTEGER" keyword? ] unit-test -{ t } [ "FLOAT" keyword? ] unit-test -{ t } [ "BOOLEAN" keyword? ] unit-test -{ t } [ "TIMESTAMP" keyword? ] unit-test -{ t } [ "JSON" keyword? ] unit-test -{ t } [ "PROMPT_SCORE" keyword? ] unit-test - -! Constraint keywords -{ t } [ "UNIQUE" keyword? ] unit-test -{ t } [ "CHECK" keyword? ] unit-test -{ t } [ "REFERENCES" keyword? ] unit-test - -! Comparison operator keywords -{ t } [ "LIKE" keyword? ] unit-test -{ t } [ "IN" keyword? ] unit-test -{ t } [ "CONTAINS" keyword? ] unit-test - -! Non-keywords (identifiers) -{ f } [ "myTable" keyword? ] unit-test -{ f } [ "users" keyword? ] unit-test -{ f } [ "evidence" keyword? ] unit-test -{ f } [ "foo_bar" keyword? ] unit-test -{ f } [ "x" keyword? ] unit-test -{ f } [ "column1" keyword? ] unit-test - -! ============================================================ -! Whitespace Skipping Tests: skip-whitespace -! ============================================================ - -{ "hello" } [ " hello" skip-whitespace ] unit-test -{ "hello" } [ "\t\thello" skip-whitespace ] unit-test -{ "hello" } [ "\n\nhello" skip-whitespace ] unit-test -{ "hello" } [ " \t\n\rhello" skip-whitespace ] unit-test -{ "hello" } [ "hello" skip-whitespace ] unit-test -{ "" } [ " " skip-whitespace ] unit-test -{ "" } [ "" skip-whitespace ] unit-test - -! ============================================================ -! Token Consumer Tests: peek-token, consume-token -! ============================================================ - -! peek-token returns first token without consuming -{ "SELECT" } [ - "SELECT * FROM t" split-tokens - peek-token -] unit-test - -! peek-token returns f on empty -{ f } [ - { } >vector peek-token -] unit-test - -! consume-token returns token and rest -{ "SELECT" } [ - "SELECT * FROM t" split-tokens - consume-token nip ! get the consumed token -] unit-test - -{ 3 } [ - "SELECT * FROM t" split-tokens - consume-token drop ! drop the consumed token, keep rest - length -] unit-test - -! ============================================================ -! expect-token Tests -! ============================================================ - -! Successful expect -{ 3 } [ - "SELECT * FROM t" split-tokens - "SELECT" expect-token - length -] unit-test - -! Case-insensitive expect -{ 3 } [ - "select * FROM t" split-tokens - "SELECT" expect-token - length -] unit-test - -! Failed expect throws error -{ t } [ - [ "INSERT * FROM t" split-tokens "SELECT" expect-token ] - [ gql-parse-error? ] recover -] unit-test - -! ============================================================ -! try-consume Tests -! ============================================================ - -! Successful try-consume -{ t } [ - "WHERE x = 1" split-tokens - "WHERE" try-consume nip ! ( tokens' matched? ) -] unit-test - -{ 3 } [ - "WHERE x = 1" split-tokens - "WHERE" try-consume drop ! matched? - length -] unit-test - -! Failed try-consume (no match) -{ f } [ - "SELECT x FROM t" split-tokens - "WHERE" try-consume nip -] unit-test - -{ 4 } [ - "SELECT x FROM t" split-tokens - "WHERE" try-consume drop - length ! tokens unchanged -] unit-test - -! try-consume on empty -{ f } [ - { } >vector - "SELECT" try-consume nip -] unit-test - -! ============================================================ -! Comment Removal Tests (parse-gql strips comments) -! ============================================================ - -! Line comments are stripped -{ t } [ - "-- this is a comment\nSELECT * FROM t" parse-gql - gql-select? -] unit-test - -{ t } [ - "SELECT * FROM t -- trailing comment" parse-gql - gql-select? -] unit-test - -! Multiple line comments -{ t } [ - "-- comment 1\n-- comment 2\nSELECT * FROM t" parse-gql - gql-select? -] unit-test - -! ============================================================ -! Semicolon Handling Tests -! ============================================================ - -! Trailing semicolons are removed -{ t } [ - "SELECT * FROM t;" parse-gql - gql-select? -] unit-test - -{ t } [ - "INSERT INTO docs { x: 1 };" parse-gql - gql-insert? -] unit-test - -! ============================================================ -! Operator Recognition Tests (via WHERE parsing) -! ============================================================ - -! Test that comparison operators are properly tokenized -{ t } [ - "SELECT * FROM t WHERE x = 1" parse-gql - where-clause>> where-clause? -] unit-test - -{ "=" } [ - "SELECT * FROM t WHERE x = 1" parse-gql - where-clause>> expression>> op>> -] unit-test - -! ============================================================ -! Navigation Path Syntax Tests -! ============================================================ - -! TRAVERSE edge_type direction -{ t } [ - "SELECT * FROM nodes TRAVERSE knows OUTBOUND" parse-gql - gql-select? -] unit-test - -! TRAVERSE with DEPTH -{ t } [ - "SELECT * FROM nodes TRAVERSE knows OUTBOUND DEPTH 3" parse-gql - gql-select? -] unit-test - -! Edge directions -{ "INBOUND" } [ - "SELECT * FROM nodes TRAVERSE follows INBOUND DEPTH 2" parse-gql - edge-clause>> direction>> -] unit-test - -{ "OUTBOUND" } [ - "SELECT * FROM nodes TRAVERSE knows OUTBOUND" parse-gql - edge-clause>> direction>> -] unit-test - -! Default depth is 1 when omitted -{ 1 } [ - "SELECT * FROM nodes TRAVERSE knows ANY" parse-gql - edge-clause>> depth>> -] unit-test - -! ============================================================ -! Literal Value Tests (via WHERE clause) -! ============================================================ - -! String value in WHERE -{ "Alice" } [ - "SELECT * FROM users WHERE name = Alice" parse-gql - where-clause>> expression>> value>> -] unit-test - -! Numeric values as strings (tokenizer yields strings) -{ "42" } [ - "SELECT * FROM data WHERE x = 42" parse-gql - where-clause>> expression>> value>> -] unit-test - -! ============================================================ -! Statement Type Recognition Tests -! ============================================================ - -{ t } [ "SELECT * FROM t" parse-gql gql-select? ] unit-test -{ t } [ "INSERT INTO t { x: 1 }" parse-gql gql-insert? ] unit-test -{ t } [ "UPDATE t SET x = 1 WHERE y = 2" parse-gql gql-update? ] unit-test -{ t } [ "DELETE FROM t WHERE x = 1" parse-gql gql-delete? ] unit-test -{ t } [ "CREATE COLLECTION t" parse-gql gql-create? ] unit-test -{ t } [ "DROP COLLECTION t" parse-gql gql-drop? ] unit-test -{ t } [ "EXPLAIN SELECT * FROM t" parse-gql gql-explain? ] unit-test -{ t } [ "INTROSPECT COLLECTIONS" parse-gql gql-introspect? ] unit-test - -! Case-insensitive statement keywords -{ t } [ "select * from t" parse-gql gql-select? ] unit-test -{ t } [ "Select * From t" parse-gql gql-select? ] unit-test -{ t } [ "insert into t { x: 1 }" parse-gql gql-insert? ] unit-test -{ t } [ "delete from t where x = 1" parse-gql gql-delete? ] unit-test - -! ============================================================ -! EXPLAIN Flag Parsing Tests -! ============================================================ - -{ t } [ - "EXPLAIN ANALYZE SELECT * FROM t" parse-gql - analyze?>> -] unit-test - -{ t } [ - "EXPLAIN VERBOSE SELECT * FROM t" parse-gql - verbose?>> -] unit-test - -{ t } [ - "EXPLAIN ANALYZE VERBOSE SELECT * FROM t" parse-gql - dup analyze?>> swap verbose?>> and -] unit-test - -! EXPLAIN without flags -{ f } [ - "EXPLAIN SELECT * FROM t" parse-gql - analyze?>> -] unit-test - -{ f } [ - "EXPLAIN SELECT * FROM t" parse-gql - verbose?>> -] unit-test - -! ============================================================ -! INTROSPECT Target Parsing Tests -! ============================================================ - -{ "COLLECTIONS" } [ - "INTROSPECT COLLECTIONS" parse-gql target>> -] unit-test - -{ "SCHEMA" } [ - "INTROSPECT SCHEMA" parse-gql target>> -] unit-test - -{ "CONSTRAINTS" } [ - "INTROSPECT CONSTRAINTS" parse-gql target>> -] unit-test - -{ "JOURNAL" } [ - "INTROSPECT JOURNAL" parse-gql target>> -] unit-test - -! JOURNAL with SINCE -{ t } [ - "INTROSPECT JOURNAL SINCE 100" parse-gql - dup target>> "JOURNAL" = - swap arg>> 100 = and -] unit-test - -! ============================================================ -! Error Recovery Tests -! ============================================================ - -! Unknown statement type throws parse error -{ t } [ - [ "SELECTT * FROM users" parse-gql ] [ gql-parse-error? ] recover -] unit-test - -{ t } [ - [ "BANANA * FROM users" parse-gql ] [ gql-parse-error? ] recover -] unit-test - -! Missing collection after FROM -{ t } [ - [ "SELECT * FROM" parse-gql drop ] - [ ] recover - ! If it doesn't crash, that's acceptable error handling - t -] unit-test - -! ============================================================ -! LIMIT / OFFSET Parsing Tests -! ============================================================ - -{ t } [ - "SELECT * FROM t LIMIT 10" parse-gql - limit-clause>> limit-clause? -] unit-test - -{ 10 } [ - "SELECT * FROM t LIMIT 10" parse-gql - limit-clause>> limit>> -] unit-test - -{ 0 } [ - "SELECT * FROM t LIMIT 10" parse-gql - limit-clause>> offset>> ! Default offset is 0 -] unit-test - -{ 20 } [ - "SELECT * FROM t LIMIT 10 OFFSET 20" parse-gql - limit-clause>> offset>> -] unit-test - -! ============================================================ -! Field List Parsing Tests -! ============================================================ - -! Star projection -{ { "*" } } [ - "SELECT * FROM t" parse-gql - fields>> >array -] unit-test - -! Named fields -{ t } [ - "SELECT name dept salary FROM users" parse-gql - fields>> length 3 = -] unit-test - -! ============================================================ -! Test Summary -! ============================================================ - -: run-lexer-tests ( -- ) - "Running GQL lexer unit tests..." print - "lexer-tests" run-tests - "Lexer tests complete." print ; - -MAIN: run-lexer-tests diff --git a/lithoglyph/core-factor/gql/seam-tests.factor b/lithoglyph/core-factor/gql/seam-tests.factor deleted file mode 100644 index 661d3215..00000000 --- a/lithoglyph/core-factor/gql/seam-tests.factor +++ /dev/null @@ -1,325 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Form.Runtime - Seam Tests -! -! End-to-end tests validating the pipeline: -! Parser -> Planner -> Executor -> Normalizer -! -! These tests verify that data flows correctly across component boundaries. - -USING: accessors arrays assocs gql fd-discovery io kernel math -namespaces sequences strings tools.test vectors ; - -IN: seam-tests - -! ============================================================ -! Test Setup -! ============================================================ - -: reset-test-state ( -- ) - collections get clear-assoc ; - -: setup-test-collection ( -- ) - reset-test-state - ! Create a test collection with sample data - "CREATE COLLECTION test_users" run-gql drop - ! Insert test documents - V{ - H{ { "id" "u1" } { "name" "Alice" } { "dept" "Engineering" } { "salary" "100000" } } - H{ { "id" "u2" } { "name" "Bob" } { "dept" "Engineering" } { "salary" "95000" } } - H{ { "id" "u3" } { "name" "Carol" } { "dept" "Sales" } { "salary" "80000" } } - H{ { "id" "u4" } { "name" "Dave" } { "dept" "Sales" } { "salary" "85000" } } - H{ { "id" "u5" } { "name" "Eve" } { "dept" "Engineering" } { "salary" "110000" } } - } "test_users" set-collection ; - -! ============================================================ -! Seam 1: Parser -> Planner -! ============================================================ - -! Test that parsed AST can be planned - -{ t } [ - "SELECT * FROM users" parse-gql - gql-select? -] unit-test - -{ t } [ - "SELECT * FROM users" parse-gql plan-query - query-plan? -] unit-test - -{ t } [ - "SELECT name, dept FROM users WHERE dept = Engineering" parse-gql plan-query - steps>> length 2 >= ! At least project + scan -] unit-test - -{ t } [ - "INSERT INTO users { name: Test }" parse-gql plan-query - steps>> first type>> "insert" = -] unit-test - -! Test EXPLAIN produces plan - -{ "ok" } [ - "EXPLAIN SELECT * FROM users" parse-gql execute-gql - "status" swap at -] unit-test - -{ t } [ - "EXPLAIN SELECT * FROM users" parse-gql execute-gql - "plan" swap at - "steps" swap at - array? -] unit-test - -! ============================================================ -! Seam 2: Planner -> Executor -! ============================================================ - -! Test that planned queries execute correctly - -{ "ok" } [ - setup-test-collection - "SELECT * FROM test_users" run-gql - "status" swap at -] unit-test - -{ 5 } [ - setup-test-collection - "SELECT * FROM test_users" run-gql - "count" swap at -] unit-test - -{ t } [ - setup-test-collection - "SELECT * FROM test_users WHERE dept = Engineering" run-gql - "count" swap at 3 = -] unit-test - -! Test INSERT -> SELECT round trip - -{ t } [ - setup-test-collection - "INSERT INTO test_users { name: Frank, dept: HR }" run-gql drop - "SELECT * FROM test_users WHERE dept = HR" run-gql - "count" swap at 1 = -] unit-test - -! Test UPDATE -> SELECT verification - -{ t } [ - setup-test-collection - "UPDATE test_users SET salary = 120000 WHERE name = Alice" run-gql drop - "SELECT salary FROM test_users WHERE name = Alice" run-gql - "rows" swap at first "salary" swap at "120000" = -] unit-test - -! Test DELETE -> SELECT verification - -{ t } [ - setup-test-collection - "DELETE FROM test_users WHERE dept = Sales" run-gql drop - "SELECT * FROM test_users" run-gql - "count" swap at 3 = ! 5 - 2 Sales employees = 3 -] unit-test - -! ============================================================ -! Seam 3: Executor -> Normalizer (FD Discovery) -! ============================================================ - -! Test that executor results can be fed to FD discovery - -{ t } [ - setup-test-collection - "test_users" get-collection :> data - data length 5 = -] unit-test - -{ t } [ - setup-test-collection - "test_users" get-collection - default-fd-config run-dfd - fd-discovery-result? -] unit-test - -! Test FD discovery on executor results - -{ t } [ - reset-test-state - ! Create collection with clear FD: employee_id -> name, dept - V{ - H{ { "emp_id" "E1" } { "name" "Alice" } { "dept" "Eng" } } - H{ { "emp_id" "E2" } { "name" "Bob" } { "dept" "Eng" } } - H{ { "emp_id" "E3" } { "name" "Carol" } { "dept" "Sales" } } - H{ { "emp_id" "E1" } { "name" "Alice" } { "dept" "Eng" } } ! Duplicate to confirm FD - } "employees" set-collection - "employees" get-collection - default-fd-config run-dfd - dependencies>> empty? not ! Should find emp_id -> name, dept -] unit-test - -! ============================================================ -! Seam 4: Full Round Trip -! ============================================================ - -! Test complete pipeline: Parse -> Plan -> Execute -> Discover -> Analyze - -: run-full-pipeline-test ( -- success? ) - reset-test-state - - ! Step 1: Create collection - "CREATE COLLECTION orders" run-gql - "status" swap at "ok" = not [ f ] [ - - ! Step 2: Insert data with clear FD pattern - V{ - H{ { "order_id" "O1" } { "customer_id" "C1" } { "customer_name" "Acme Corp" } { "total" "1000" } } - H{ { "order_id" "O2" } { "customer_id" "C1" } { "customer_name" "Acme Corp" } { "total" "2000" } } - H{ { "order_id" "O3" } { "customer_id" "C2" } { "customer_name" "Beta Inc" } { "total" "1500" } } - H{ { "order_id" "O4" } { "customer_id" "C2" } { "customer_name" "Beta Inc" } { "total" "3000" } } - } "orders" set-collection - - ! Step 3: Query and verify - "SELECT * FROM orders" run-gql - "count" swap at 4 = not [ f ] [ - - ! Step 4: Run FD discovery - "orders" get-collection - default-fd-config run-dfd :> fd-result - - ! Step 5: Check normal form - fd-result dependencies>> :> fds - { { "order_id" } } :> keys ! order_id is the key - fds keys analyze-normal-form :> nf-analysis - - ! Step 6: Verify we can generate narrative - fd-result result>narrative length 0 > - - ] if - ] if ; - -{ t } [ - run-full-pipeline-test -] unit-test - -! ============================================================ -! Seam 5: EXPLAIN -> Execution Correlation -! ============================================================ - -! Test that EXPLAIN ANALYZE produces timing data - -{ t } [ - setup-test-collection - "EXPLAIN ANALYZE SELECT * FROM test_users" run-gql - "execution_time_ms" swap key? -] unit-test - -{ t } [ - setup-test-collection - "EXPLAIN ANALYZE SELECT * FROM test_users" run-gql - "actual_result" swap at - "count" swap at 5 = -] unit-test - -! Test EXPLAIN VERBOSE produces readable plan - -{ t } [ - setup-test-collection - "EXPLAIN VERBOSE SELECT * FROM test_users WHERE dept = Engineering" run-gql - "verbose_plan" swap at - "Seq Scan" swap subseq? -] unit-test - -! ============================================================ -! Seam 6: Introspection -> FD Discovery -! ============================================================ - -! Test that introspection data can be used for FD discovery context - -{ t } [ - setup-test-collection - "INTROSPECT COLLECTIONS" run-gql - "collections" swap at - "test_users" swap member? -] unit-test - -{ "ok" } [ - setup-test-collection - "INTROSPECT SCHEMA" run-gql - "status" swap at -] unit-test - -! ============================================================ -! Error Propagation Tests -! ============================================================ - -! Test that parser errors don't crash planner - -{ t } [ - [ "SELECTT * FROM users" parse-gql ] [ gql-parse-error? ] recover -] unit-test - -! Test that missing collection returns error gracefully - -{ "ok" } [ - reset-test-state - "SELECT * FROM nonexistent" run-gql - "status" swap at - ! Should still return ok, just with empty results -] unit-test - -{ 0 } [ - reset-test-state - "SELECT * FROM nonexistent" run-gql - "count" swap at -] unit-test - -! ============================================================ -! Stress Test: Large Dataset -! ============================================================ - -: create-large-dataset ( n -- ) - reset-test-state - V{ } clone swap [ - [ ] dip ! ( accum i ) - H{ } clone - over "id" number>string "U" prepend "id" pick set-at - over 2 mod 0 = [ "Eng" ] [ "Sales" ] if "dept" pick set-at - over 50000 + number>string "salary" pick set-at - nip swap push - ] each-integer - "large_users" set-collection ; - -{ t } [ - 1000 create-large-dataset - "SELECT * FROM large_users" run-gql - "count" swap at 1000 = -] unit-test - -{ t } [ - 1000 create-large-dataset - "SELECT * FROM large_users WHERE dept = Eng" run-gql - "count" swap at 500 = -] unit-test - -{ t } [ - 1000 create-large-dataset - "large_users" get-collection - fd-discovery-config new - 100 >>sample-size ! Sample for speed - 0.95 >>confidence-threshold - "dfd" >>algorithm - 3 >>max-lhs-size - run-dfd - fd-discovery-result? -] unit-test - -! ============================================================ -! Test Summary -! ============================================================ - -: run-seam-tests ( -- ) - "Running seam tests (Parser -> Planner -> Executor -> Normalizer)..." print - "seam-tests" run-tests - "Seam tests complete." print ; - -MAIN: run-seam-tests diff --git a/lithoglyph/core-factor/gql/storage-backend.factor b/lithoglyph/core-factor/gql/storage-backend.factor deleted file mode 100644 index 640b15aa..00000000 --- a/lithoglyph/core-factor/gql/storage-backend.factor +++ /dev/null @@ -1,471 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Form.Runtime - Storage Backend Abstraction -! -! Pluggable storage layer for GQL executor. -! - memory: In-memory storage (default, for testing) -! - bridge: Persistent storage via Form.Bridge (production) - -USING: accessors alien alien.c-types alien.data alien.libraries alien.strings -arrays assocs byte-arrays classes.struct combinators formatting hashtables -io io.encodings.utf8 json.reader json.writer kernel locals math namespaces -sequences strings vectors ; - -IN: storage-backend - -! ============================================================ -! Lithoglyph Bridge FFI Library -! ============================================================ - -<< "lithoglyph-bridge" { - { [ os linux? ] [ "libbridge.so" ] } - { [ os macosx? ] [ "libbridge.dylib" ] } - { [ os windows? ] [ "bridge.dll" ] } -} cond cdecl add-library >> - -! ============================================================ -! Storage Backend Protocol -! ============================================================ - -MIXIN: storage-backend - -GENERIC: backend-init ( backend -- ) -GENERIC: backend-close ( backend -- ) -GENERIC: backend-get-collection ( name backend -- data ) -GENERIC: backend-set-collection ( data name backend -- ) -GENERIC: backend-list-collections ( backend -- names ) -GENERIC: backend-delete-collection ( name backend -- ) -GENERIC: backend-insert ( doc collection backend -- id ) -GENERIC: backend-update ( doc id collection backend -- success? ) -GENERIC: backend-delete ( id collection backend -- success? ) -GENERIC: backend-query ( collection backend -- rows ) - -! ============================================================ -! In-Memory Backend (Default) -! ============================================================ - -TUPLE: memory-backend - collections - next-id ; - -: ( -- backend ) - memory-backend new - H{ } clone >>collections - 1 >>next-id ; - -INSTANCE: memory-backend storage-backend - -M: memory-backend backend-init - drop ; - -M: memory-backend backend-close - drop ; - -M: memory-backend backend-get-collection - collections>> at [ V{ } clone ] unless* ; - -M: memory-backend backend-set-collection - collections>> set-at ; - -M: memory-backend backend-list-collections - collections>> keys ; - -M: memory-backend backend-delete-collection - collections>> delete-at ; - -M:: memory-backend backend-insert ( doc collection backend -- id ) - backend next-id>> :> id - id 1 + backend next-id<< - ! Add id to document - id number>string "_id" doc set-at - ! Get or create collection - collection backend backend-get-collection :> coll - doc coll push - coll collection backend backend-set-collection - id ; - -M:: memory-backend backend-update ( doc id collection backend -- success? ) - collection backend backend-get-collection :> coll - f :> found! - coll [ - dup "_id" swap at id number>string = [ - drop doc t found! - ] when - ] map collection backend backend-set-collection - found ; - -M:: memory-backend backend-delete ( id collection backend -- success? ) - collection backend backend-get-collection :> coll - coll [ - "_id" swap at id number>string = not - ] filter :> new-coll - new-coll length coll length < :> deleted? - new-coll collection backend backend-set-collection - deleted? ; - -M: memory-backend backend-query - backend-get-collection ; - -! ============================================================ -! Bridge Backend (Persistent Storage) -! ============================================================ - -! FFI type definitions matching generated/abi/bridge.h -STRUCT: lith-blob - { ptr void* } - { len size_t } ; - -! LgResult — matches bridge.h LgResult struct layout -STRUCT: lith-result - { data lith-blob } - { provenance lith-blob } - { status int } - { error_blob lith-blob } ; - -! LgRenderOpts — matches bridge.h -STRUCT: lg-render-opts - { format int } - { include_metadata bool } ; - -! Status codes matching bridge.h LithStatus enum -CONSTANT: LITH_OK 0 -CONSTANT: LITH_ERR_INTERNAL 1 -CONSTANT: LITH_ERR_NOT_FOUND 2 -CONSTANT: LITH_ERR_INVALID_ARGUMENT 3 -CONSTANT: LITH_ERR_OUT_OF_MEMORY 4 -CONSTANT: LITH_ERR_NOT_IMPLEMENTED 5 -CONSTANT: LITH_ERR_TXN_NOT_ACTIVE 6 -CONSTANT: LITH_ERR_TXN_ALREADY_COMMITTED 7 -CONSTANT: LITH_ERR_IO_ERROR 8 -CONSTANT: LITH_ERR_CORRUPTION 9 -CONSTANT: LITH_ERR_CONFLICT 10 -CONSTANT: LITH_ERR_ALREADY_EXISTS 11 - -! Transaction mode constants matching bridge.h LgTxnMode -CONSTANT: LG_TXN_READ_ONLY 0 -CONSTANT: LG_TXN_READ_WRITE 1 - -! ============================================================ -! FFI Function Declarations (from generated/abi/bridge.h) -! ============================================================ - -LIBRARY: lithoglyph-bridge - -! Database lifecycle -FUNCTION: int lith_db_open ( void* path ulong path_len void* opts ulong opts_len void** out_db lith-blob* out_err ) -FUNCTION: int lith_db_close ( void* db ) -FUNCTION: uint lith_version ( ) - -! Transaction management -FUNCTION: int lith_txn_begin ( void* db int mode void** out_txn lith-blob* out_err ) -FUNCTION: int lith_txn_commit ( void* txn lith-blob* out_err ) -FUNCTION: int lith_txn_abort ( void* txn ) - -! Operations (buffered until commit) -FUNCTION: lith-result lith_apply ( void* txn void* op ulong op_len ) -FUNCTION: int lith_update_block ( void* txn ulong block_id void* data ulong data_len lith-blob* out_err ) -FUNCTION: int lith_delete_block ( void* txn ulong block_id lith-blob* out_err ) - -! Query (full block scan) -FUNCTION: int lith_read_blocks ( void* db ushort block_type lith-blob* out_data lith-blob* out_err ) - -! Introspection -FUNCTION: int lith_introspect_schema ( void* db lith-blob* out_schema lith-blob* out_err ) -FUNCTION: int lith_introspect_constraints ( void* db lith-blob* out_constraints lith-blob* out_err ) -FUNCTION: int lith_render_journal ( void* db ulong since lg-render-opts opts lith-blob* out_text lith-blob* out_err ) -FUNCTION: int lith_render_block ( void* db ulong block_id lg-render-opts opts lith-blob* out_text lith-blob* out_err ) - -! Proof verification -FUNCTION: int lith_proof_register_verifier ( void* type_ptr ulong type_len void* callback void* context ) -FUNCTION: int lith_proof_unregister_verifier ( void* type_ptr ulong type_len ) -FUNCTION: int lith_proof_verify ( void* proof_ptr ulong proof_len bool* out_valid lith-blob* out_err ) -FUNCTION: int lith_proof_init_builtins ( ) - -! Resource cleanup -FUNCTION: void lith_blob_free ( lith-blob* blob ) - -! ============================================================ -! FFI Helper Functions -! ============================================================ - -: make-lith-blob ( -- blob ) - lith-blob malloc-struct - f >>ptr - 0 >>len ; - -: blob>string ( blob -- str/f ) - dup [ ptr>> ] [ len>> ] bi over [ - memory>byte-array utf8 decode - ] [ 2drop f ] if ; - -: string>lith-input ( str -- ptr len ) - utf8 encode [ underlying>> ] [ length ] bi ; - -: check-lith-status ( status err-blob -- ) - swap LITH_OK = [ - drop - ] [ - blob>string "FFI Error: %s\n" sprintf throw - ] if ; - -! Block type constant for documents (0x0011) -CONSTANT: BLOCK_TYPE_DOCUMENT 0x0011 - -! Parse JSON array of block results into Factor vector of hashtables. -! Input: JSON string like [{"block_id":1,"size":42,"data":"..."},...] -! Output: Vector of hashtables (the "data" field is parsed as JSON if possible) -: parse-block-results ( json-str -- vec ) - json> dup array? [ - >vector - [ - dup hashtable? [ - ! Try to parse the "data" field as JSON - dup "data" swap at [ - [ json> ] [ drop f ] recover - dup hashtable? [ - ! Successfully parsed document JSON - swap "data" pick set-at - ] [ drop ] if - ] when* - ] when - ] map - ] [ drop V{ } clone ] if ; - -! ============================================================ -! Bridge Transaction Helper -! ============================================================ - -! Execute a single-operation transaction: begin → apply → commit -! Returns the block ID from lith_apply result, or 0 on failure. -: with-bridge-txn ( doc-json backend -- block-id ) - db-handle>> :> db - f :> txn-handle! - make-lith-blob :> err-blob - - ! Begin read-write transaction (mode 1 = read-write) - db LG_TXN_READ_WRITE txn-handle! err-blob lith_txn_begin - err-blob check-lith-status - - ! Apply the operation (buffered, not written until commit) - swap string>lith-input [ - txn-handle lith_apply - ] 2keep 2drop :> result - - ! Commit transaction (WAL: journal → sync → blocks → sync) - txn-handle err-blob lith_txn_commit - err-blob check-lith-status - - ! Extract block_id from result (JSON: {"block_id":N,"status":"pending"}) - result data>> blob>string [ - json> dup hashtable? [ - "block_id" swap at [ 0 ] unless* - ] [ drop 0 ] if - ] [ 0 ] if* ; - -! ============================================================ -! Bridge Backend Tuple -! ============================================================ - -TUPLE: bridge-backend - db-handle - db-path - is-open ; - -: ( path -- backend ) - bridge-backend new - swap >>db-path - f >>db-handle - f >>is-open ; - -INSTANCE: bridge-backend storage-backend - -! Bridge backend methods - FFI implementations - -M:: backend-init ( backend -- ) bridge-backend - backend db-path>> :> path - f :> db-handle! - make-lith-blob :> err-blob - - path string>lith-input f 0 { void* } [ - db-handle! err-blob lith_db_open - ] with-out-parameters - - err-blob check-lith-status - db-handle backend db-handle<< - t backend is-open<< - "Database opened: %s\n" path sprintf print ; - -M:: bridge-backend backend-close ( backend -- ) - backend is-open>> backend db-handle>> and [ - backend db-handle>> lith_db_close - f backend db-handle<< - f backend is-open<< - "Database closed\n" print - ] when ; - -M:: bridge-backend backend-get-collection ( name backend -- data ) - backend db-handle>> [ - make-lith-blob :> data-blob - make-lith-blob :> err-blob - - ! Read all document blocks via lith_read_blocks (type = 0x0011) - backend db-handle>> BLOCK_TYPE_DOCUMENT data-blob err-blob lith_read_blocks - err-blob check-lith-status - - ! Parse JSON result into vector of documents - data-blob blob>string [ "[]" ] unless* - parse-block-results - - ! Free the blob - data-blob lith_blob_free - ] [ V{ } clone ] if ; - -M:: bridge-backend backend-set-collection ( data name backend -- ) - backend db-handle>> [ - ! Insert each document via individual transactions - data [ - >json backend with-bridge-txn drop - ] each - ] when ; - -M:: bridge-backend backend-list-collections ( backend -- names ) - backend db-handle>> [ - make-lith-blob :> schema-blob - make-lith-blob :> err-blob - - backend db-handle>> schema-blob err-blob lith_introspect_schema - err-blob check-lith-status - - ! Parse JSON schema and extract collection names - schema-blob blob>string [ "{\"version\":0,\"collections\":[]}" ] unless* - json> dup hashtable? [ - "collections" swap at [ { } ] unless* - ] [ drop { } ] if - - schema-blob lith_blob_free - ] [ { } ] if ; - -M:: bridge-backend backend-delete-collection ( name backend -- ) - backend db-handle>> [ - ! Read all blocks and delete those matching the collection - ! (For PoC, collection filtering is not yet implemented at block level) - "Bridge backend: delete-collection %s (requires collection metadata)\n" - name sprintf print - ] when ; - -M:: bridge-backend backend-insert ( doc collection backend -- id ) - backend db-handle>> [ - ! Serialize document to JSON for storage - doc >json :> doc-json - - ! Execute insert through bridge transaction - doc-json backend with-bridge-txn - ] [ 0 ] if ; - -M:: bridge-backend backend-update ( doc id collection backend -- success? ) - backend db-handle>> [ - f :> txn-handle! - make-lith-blob :> err-blob - - ! Begin read-write transaction - backend db-handle>> LG_TXN_READ_WRITE txn-handle! err-blob lith_txn_begin - err-blob check-lith-status - - ! Serialize new document data - doc >json :> doc-json - doc-json string>lith-input :> ( data-ptr data-len ) - - ! Update the block - txn-handle id data-ptr data-len err-blob lith_update_block - err-blob check-lith-status - - ! Commit - txn-handle err-blob lith_txn_commit - err-blob check-lith-status - t - ] [ f ] if ; - -M:: bridge-backend backend-delete ( id collection backend -- success? ) - backend db-handle>> [ - f :> txn-handle! - make-lith-blob :> err-blob - - ! Begin read-write transaction - backend db-handle>> LG_TXN_READ_WRITE txn-handle! err-blob lith_txn_begin - err-blob check-lith-status - - ! Delete the block - txn-handle id err-blob lith_delete_block - err-blob check-lith-status - - ! Commit - txn-handle err-blob lith_txn_commit - err-blob check-lith-status - t - ] [ f ] if ; - -M: bridge-backend backend-query - backend-get-collection ; - -! ============================================================ -! Global Backend Selection -! ============================================================ - -SYMBOL: current-backend - -: init-memory-backend ( -- ) - dup backend-init current-backend set ; - -: init-bridge-backend ( path -- ) - dup backend-init current-backend set ; - -: get-backend ( -- backend ) - current-backend get [ init-memory-backend current-backend get ] unless* ; - -: close-backend ( -- ) - current-backend get [ backend-close ] when* - f current-backend set ; - -! ============================================================ -! Convenience API (used by executor) -! ============================================================ - -: storage-get-collection ( name -- data ) - get-backend backend-get-collection ; - -: storage-set-collection ( data name -- ) - get-backend backend-set-collection ; - -: storage-list-collections ( -- names ) - get-backend backend-list-collections ; - -: storage-delete-collection ( name -- ) - get-backend backend-delete-collection ; - -: storage-insert ( doc collection -- id ) - get-backend backend-insert ; - -: storage-update ( doc id collection -- success? ) - get-backend backend-update ; - -: storage-delete ( id collection -- success? ) - get-backend backend-delete ; - -: storage-query ( collection -- rows ) - get-backend backend-query ; - -! ============================================================ -! Backend Selection at Startup -! ============================================================ - -: use-memory-storage ( -- ) - close-backend - init-memory-backend - "Using in-memory storage backend\n" print ; - -: use-bridge-storage ( path -- ) - close-backend - init-bridge-backend - "Using bridge storage backend\n" print ; - -! Default to memory backend -init-memory-backend diff --git a/lithoglyph/core-factor/minimal-ffi-test.factor b/lithoglyph/core-factor/minimal-ffi-test.factor deleted file mode 100644 index 15864c86..00000000 --- a/lithoglyph/core-factor/minimal-ffi-test.factor +++ /dev/null @@ -1,25 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Minimal FFI test - just test version function - -USING: alien alien.c-types alien.libraries kernel io ; - -IN: minimal-ffi-test - -! Load library -<< "lithoglyph-bridge" { - { [ os linux? ] [ "core-factor/libbridge.so" ] } - { [ os macosx? ] [ "core-factor/libbridge.dylib" ] } - { [ os windows? ] [ "core-factor/bridge.dll" ] } -} cond cdecl add-library >> - -! Declare simplest function -LIBRARY: lithoglyph-bridge -FUNCTION: int lith_get_version ( ) - -! Test it -: test-version ( -- ) - "Testing lith_get_version..." print - lith_get_version "Version: " write . ; - -! Run test -test-version diff --git a/lithoglyph/core-factor/test-ffi.factor b/lithoglyph/core-factor/test-ffi.factor deleted file mode 100644 index d3fc90a6..00000000 --- a/lithoglyph/core-factor/test-ffi.factor +++ /dev/null @@ -1,42 +0,0 @@ -! SPDX-License-Identifier: MPL-2.0 -! Test FFI bindings to Lithoglyph bridge - -USING: io io.encodings.utf8 io.files kernel sequences storage-backend ; - -IN: test-ffi - -! Test 1: Initialize in-memory backend (should work) -: test-memory-backend ( -- ) - "=== Test 1: Memory Backend ===" print - use-memory-storage - "users" V{ } clone storage-set-collection - "users" storage-get-collection length . - "✅ Memory backend works\n" print ; - -! Test 2: Initialize bridge backend -: test-bridge-backend ( -- ) - "=== Test 2: Bridge Backend (FFI) ===" print - "test-ffi.lgh" use-bridge-storage - storage-list-collections . - "✅ Bridge backend initialized\n" print ; - -! Test 3: Insert document -: test-insert ( -- ) - "=== Test 3: Insert Document ===" print - H{ - { "name" "Alice" } - { "email" "alice@example.com" } - } "users" storage-insert . - "✅ Document inserted\n" print ; - -! Run all tests -: run-ffi-tests ( -- ) - [ - test-memory-backend - test-bridge-backend - test-insert - close-backend - "All tests completed!\n" print - ] [ - "Test failed: " write print - ] recover ; diff --git a/lithoglyph/core-forth/README.adoc b/lithoglyph/core-forth/README.adoc deleted file mode 100644 index 51f75ca5..00000000 --- a/lithoglyph/core-forth/README.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= Form.Blocks + Form.Model (Forth) -:toc: - -The truth core of Lith, implemented in Forth. - -== Form.Blocks - -Deterministic storage layer providing: - -* Fixed-size blocks with symbolic headers -* Append-only journal -* Crash recovery -* Integrity checks + repair guidance - -== Form.Model - -Multi-model logical layer providing: - -* Document collections -* Edge collections -* Schema + constraint metadata -* Migration artefacts - -== Directory Structure - -[source,text] ----- -core-forth/ -├── Form.Blocks/ # Storage primitives -├── Form.Model/ # Logical layer -└── tests/ # Unit tests ----- - -== Status - -Implementation pending specification completion. diff --git a/lithoglyph/core-forth/src/lithoglyph-blocks.fs b/lithoglyph/core-forth/src/lithoglyph-blocks.fs deleted file mode 100644 index 2e2cc7ea..00000000 --- a/lithoglyph/core-forth/src/lithoglyph-blocks.fs +++ /dev/null @@ -1,355 +0,0 @@ -\ SPDX-License-Identifier: MPL-2.0 -\ Form.Blocks - Fixed-size block storage layer -\ -\ This is the truth core of Lithoglyph. All data goes through here. -\ No business logic - just blocks, journals, and integrity. -\ -\ Lithoglyph = litho (stone) + glyph (carved symbol) -\ Forth sculpts data onto disk like carving glyphs in stone. - -\ Reset to clean Forth vocabulary to avoid gforth BLOCKS extension conflicts -only forth definitions - -\ ============================================================ -\ Constants -\ ============================================================ - -4096 constant BLOCK-SIZE \ 4 KiB blocks -64 constant HEADER-SIZE \ 64-byte fixed header -BLOCK-SIZE HEADER-SIZE - constant PAYLOAD-SIZE \ 4032 bytes - -\ Magic bytes: "LGH\0" = 0x4C474800 (Lithoglyph) -$4C474800 constant BLOCK-MAGIC - -\ Block types -$0000 constant TYPE-FREE -$0001 constant TYPE-SUPERBLOCK -$0010 constant TYPE-COLLECTION-META -$0011 constant TYPE-DOCUMENT -$0012 constant TYPE-DOCUMENT-OVERFLOW -$0020 constant TYPE-EDGE-META -$0021 constant TYPE-EDGE -$0030 constant TYPE-INDEX-ROOT -$0031 constant TYPE-INDEX-INTERNAL -$0032 constant TYPE-INDEX-LEAF -$0040 constant TYPE-JOURNAL-SEGMENT -$0050 constant TYPE-SCHEMA -$0051 constant TYPE-CONSTRAINT -$0060 constant TYPE-MIGRATION - -\ Block flags (bitmask) -$01 constant FLAG-COMPRESSED -$02 constant FLAG-ENCRYPTED -$04 constant FLAG-CHAINED -$08 constant FLAG-DELETED - -\ ============================================================ -\ Block Header Structure -\ ============================================================ -\ Offset Size Field -\ 0 4 magic -\ 4 2 version -\ 6 2 block_type -\ 8 8 block_id -\ 16 8 sequence -\ 24 8 created_at -\ 32 8 modified_at -\ 40 4 payload_len -\ 44 4 checksum -\ 48 8 prev_block_id -\ 56 4 flags -\ 60 4 reserved - -\ Header field accessors (offset from block start) -: blk-magic ( addr -- addr' ) ; -: block-version ( addr -- addr' ) 4 + ; -: block-type ( addr -- addr' ) 6 + ; -: block-id ( addr -- addr' ) 8 + ; -: block-sequence ( addr -- addr' ) 16 + ; -: block-created ( addr -- addr' ) 24 + ; -: block-modified ( addr -- addr' ) 32 + ; -: block-payload-len ( addr -- addr' ) 40 + ; -: block-checksum ( addr -- addr' ) 44 + ; -: block-prev ( addr -- addr' ) 48 + ; -: block-flags ( addr -- addr' ) 56 + ; -: block-reserved ( addr -- addr' ) 60 + ; -: block-payload ( addr -- addr' ) HEADER-SIZE + ; - -\ ============================================================ -\ Memory Buffer for Block Operations -\ ============================================================ - -create block-buffer BLOCK-SIZE allot - -\ Initialize block buffer to zeros -: clear-block ( -- ) - block-buffer BLOCK-SIZE 0 fill ; - -\ ============================================================ -\ CRC32C Implementation (Castagnoli polynomial) -\ ============================================================ - -\ CRC32C lookup table -create crc32c-table 256 cells allot - -\ Initialize CRC32C table (Castagnoli polynomial: 0x1EDC6F41) -: init-crc32c-table ( -- ) - 256 0 do - i - 8 0 do - dup 1 and if - 1 rshift $82F63B78 xor - else - 1 rshift - then - loop - crc32c-table i cells + ! - loop ; - -\ Calculate CRC32C of memory region -\ Use a variable to hold the address since DO/LOOP uses return stack -variable crc-addr -: crc32c ( addr len -- crc ) - swap crc-addr ! \ save addr; stack: len - $FFFFFFFF swap \ stack: crc len - 0 ?do \ use ?do to handle len=0; stack: crc - crc-addr @ i + c@ \ read byte at addr+i; stack: crc byte - over $FF and xor \ XOR with low byte of crc; stack: crc index - cells crc32c-table + @ \ lookup in table; stack: crc table-value - swap 8 rshift xor \ combine: (crc >> 8) XOR table-value; stack: new-crc - loop - $FFFFFFFF xor ; \ final XOR - -\ ============================================================ -\ Block Validation -\ ============================================================ - -\ Check if block has valid magic bytes -: valid-magic? ( addr -- flag ) - blk-magic l@ BLOCK-MAGIC = ; - -\ Check if block type is known -: valid-type? ( addr -- flag ) - block-type w@ - dup TYPE-FREE = swap - dup TYPE-SUPERBLOCK = swap - dup TYPE-DOCUMENT = swap - dup TYPE-EDGE = swap - dup TYPE-JOURNAL-SEGMENT = swap - drop - or or or or ; - -\ Check payload length is within bounds -: valid-payload-len? ( addr -- flag ) - block-payload-len l@ PAYLOAD-SIZE <= ; - -\ Compute and verify checksum -: valid-checksum? ( addr -- flag ) - dup block-payload swap block-payload-len l@ - crc32c - swap block-checksum l@ = ; - -\ Full block validation -: validate-block ( addr -- flag ) - dup valid-magic? 0= if drop false exit then - dup valid-type? 0= if drop false exit then - dup valid-payload-len? 0= if drop false exit then - valid-checksum? ; - -\ ============================================================ -\ Block Creation -\ ============================================================ - -variable next-block-id -variable current-sequence - -\ Get current timestamp (Unix microseconds) -\ gforth's utime returns double-cell microseconds since epoch -: now-microseconds ( -- u ) - utime drop ; \ On 64-bit, low cell holds full value - -\ Initialize a new block header -\ Note: Use l! for 32-bit fields, w! for 16-bit, ! for 64-bit -: init-block-header ( type block-id -- ) - clear-block - block-buffer block-id ! \ 64-bit - block-buffer block-type w! \ 16-bit - BLOCK-MAGIC block-buffer blk-magic l! \ 32-bit magic - 1 block-buffer block-version w! \ 16-bit version - now-microseconds block-buffer block-created ! \ 64-bit - now-microseconds block-buffer block-modified ! \ 64-bit - current-sequence @ block-buffer block-sequence ! \ 64-bit - 0 block-buffer block-prev ! \ 64-bit - 0 block-buffer block-flags l! \ 32-bit - 0 block-buffer block-reserved l! \ 32-bit - 0 block-buffer block-payload-len l! ; \ 32-bit - -\ Set block payload and compute checksum -: set-block-payload ( src-addr len -- ) - dup PAYLOAD-SIZE > if - drop drop - ." Error: payload too large" cr - exit - then - dup block-buffer block-payload-len l! \ 32-bit store - block-buffer block-payload swap move - \ Compute checksum - block-buffer block-payload - block-buffer block-payload-len l@ \ 32-bit fetch - crc32c - block-buffer block-checksum l! ; \ 32-bit store - -\ ============================================================ -\ Block I/O -\ ============================================================ - -variable db-file-id - -\ Open database file -: open-db ( addr len -- flag ) - r/w open-file if - drop false - else - db-file-id ! true - then ; - -\ Close database file -: close-db ( -- ) - db-file-id @ close-file drop ; - -\ Flush database file to disk (ensure durability) -: flush-db ( -- ) - db-file-id @ flush-file drop ; - -\ Read block from file -: read-block ( block-id addr -- flag ) - swap BLOCK-SIZE * db-file-id @ reposition-file if - drop false exit - then - drop - BLOCK-SIZE db-file-id @ read-file if - drop false - else - BLOCK-SIZE = - then ; - -\ Write block to file -: write-block ( block-id addr -- flag ) - swap BLOCK-SIZE * db-file-id @ reposition-file if - drop false exit - then - drop - BLOCK-SIZE db-file-id @ write-file if - drop false - else - drop true - then ; - -\ ============================================================ -\ Superblock Operations -\ ============================================================ - -create superblock-uuid 16 allot -variable superblock-journal-head -variable superblock-checkpoint -variable superblock-total-blocks -variable superblock-free-blocks -create superblock-name 64 allot - -\ Initialize superblock -: init-superblock ( -- ) - TYPE-SUPERBLOCK 0 init-block-header - \ UUID would be generated here - 0 superblock-journal-head ! - 0 superblock-checkpoint ! - 1 superblock-total-blocks ! - 0 superblock-free-blocks ! ; - -\ Read superblock from database -: read-superblock ( -- flag ) - 0 block-buffer read-block if - block-buffer validate-block if - block-buffer block-type w@ TYPE-SUPERBLOCK = if - \ Parse superblock payload - true - else - false - then - else - false - then - else - false - then ; - -\ Write superblock to database -: write-superblock ( -- flag ) - 0 block-buffer write-block ; - -\ ============================================================ -\ Block Allocation -\ ============================================================ - -\ Allocate new block ID -: alloc-block-id ( -- id ) - next-block-id @ - 1 next-block-id +! - 1 superblock-total-blocks +! ; - -\ Free a block (mark as deleted) -: free-block ( block-id -- ) - block-buffer read-block if - FLAG-DELETED block-buffer block-flags l@ or - block-buffer block-flags l! - block-buffer write-block drop - 1 superblock-free-blocks +! - then ; - -\ ============================================================ -\ Canonical Rendering -\ ============================================================ - -\ Render block type as string -: .block-type ( type -- ) - case - TYPE-FREE of ." FREE" endof - TYPE-SUPERBLOCK of ." SUPERBLOCK" endof - TYPE-DOCUMENT of ." DOCUMENT" endof - TYPE-EDGE of ." EDGE" endof - TYPE-JOURNAL-SEGMENT of ." JOURNAL_SEGMENT" endof - ." UNKNOWN" - endcase ; - -\ Render block flags -: .block-flags ( flags -- ) - ." [" - dup FLAG-COMPRESSED and if ." COMPRESSED " then - dup FLAG-ENCRYPTED and if ." ENCRYPTED " then - dup FLAG-CHAINED and if ." CHAINED " then - dup FLAG-DELETED and if ." DELETED " then - drop - ." ]" ; - -\ Render block header (canonical format) -: .block-header ( addr -- ) - ." BLOCK block_id=" dup block-id @ . - ." version=" dup block-version w@ . - ." type=" dup block-type w@ .block-type cr - ." sequence=" dup block-sequence @ . - ." created=" dup block-created @ . cr - ." payload_len=" dup block-payload-len l@ . - ." checksum=0x" dup block-checksum l@ hex . decimal cr - ." flags=" block-flags l@ .block-flags cr ; - -\ ============================================================ -\ Initialization -\ ============================================================ - -: init-blocks ( -- ) - init-crc32c-table - 0 next-block-id ! - 0 current-sequence ! - clear-block ; - -\ Entry point -init-blocks diff --git a/lithoglyph/core-forth/src/lithoglyph-journal.fs b/lithoglyph/core-forth/src/lithoglyph-journal.fs deleted file mode 100644 index cba93bd7..00000000 --- a/lithoglyph/core-forth/src/lithoglyph-journal.fs +++ /dev/null @@ -1,473 +0,0 @@ -\ SPDX-License-Identifier: MPL-2.0 -\ Form.Journal - Append-only mutation log -\ -\ Every mutation is journaled before being applied. -\ The journal enables crash recovery and provides provenance. -\ -\ Part of Lithoglyph: Stone-carved data for the ages. - -require lithoglyph-blocks.fs - -\ ============================================================ -\ Journal Constants -\ ============================================================ - -\ Journal magic: 0x4644424A (legacy name "FDBJ", retained for format compatibility) -$4644424A constant JOURNAL-MAGIC - -\ Entry header size: 48 bytes -48 constant ENTRY-HEADER-SIZE - -\ Operation types -$0001 constant OP-DOC-INSERT -$0002 constant OP-DOC-UPDATE -$0003 constant OP-DOC-DELETE -$0004 constant OP-DOC-REPLACE -$0010 constant OP-EDGE-INSERT -$0011 constant OP-EDGE-DELETE -$0012 constant OP-EDGE-UPDATE -$0020 constant OP-COLLECTION-CREATE -$0021 constant OP-COLLECTION-DROP -$0030 constant OP-SCHEMA-CREATE -$0031 constant OP-SCHEMA-ALTER -$0040 constant OP-CONSTRAINT-ADD -$0041 constant OP-CONSTRAINT-DROP -$0050 constant OP-INDEX-CREATE -$0051 constant OP-INDEX-DROP -$0060 constant OP-MIGRATION-START -$0061 constant OP-MIGRATION-STEP -$0062 constant OP-MIGRATION-COMPLETE -$0063 constant OP-MIGRATION-ROLLBACK -$0070 constant OP-CHECKPOINT -$FF00 constant OP-IRREVERSIBLE - -\ Entry flags -$01 constant EFLAG-COMMITTED -$02 constant EFLAG-ROLLED-BACK -$04 constant EFLAG-CHECKPOINT -$08 constant EFLAG-COMPRESSED -$10 constant EFLAG-IRREVERSIBLE - -\ ============================================================ -\ Journal Entry Structure -\ ============================================================ -\ Offset Size Field -\ 0 8 sequence -\ 8 8 timestamp -\ 16 2 op_type -\ 18 2 flags -\ 20 4 forward_len -\ 24 4 inverse_len -\ 28 4 provenance_len -\ 32 8 affected_block -\ 40 4 checksum -\ 44 4 entry_len - -\ Entry header field accessors -: entry-sequence ( addr -- addr' ) ; -: entry-timestamp ( addr -- addr' ) 8 + ; -: entry-op-type ( addr -- addr' ) 16 + ; -: entry-flags ( addr -- addr' ) 18 + ; -: entry-forward-len ( addr -- addr' ) 20 + ; -: entry-inverse-len ( addr -- addr' ) 24 + ; -: entry-provenance-len ( addr -- addr' ) 28 + ; -: entry-affected ( addr -- addr' ) 32 + ; -: entry-checksum ( addr -- addr' ) 40 + ; -: entry-len ( addr -- addr' ) 44 + ; -: entry-forward ( addr -- addr' ) ENTRY-HEADER-SIZE + ; - -\ Get inverse payload address -: entry-inverse ( addr -- addr' ) - dup entry-forward swap entry-forward-len @ + ; - -\ Get provenance payload address -: entry-provenance ( addr -- addr' ) - dup entry-inverse swap entry-inverse-len @ + ; - -\ ============================================================ -\ Journal State -\ ============================================================ - -variable journal-file-id -variable journal-sequence \ Current sequence number -variable journal-head \ Last committed sequence -variable journal-checkpoint \ Last checkpoint sequence -variable journal-entry-count -variable journal-file-size - -\ Journal entry buffer (max 64KB per entry) -65536 constant MAX-ENTRY-SIZE -create entry-buffer MAX-ENTRY-SIZE allot - -\ ============================================================ -\ Journal File Operations -\ ============================================================ - -\ Open journal file -: open-journal ( addr len -- flag ) - r/w open-file if - drop false - else - journal-file-id ! true - then ; - -\ Create new journal file -: create-journal ( addr len -- flag ) - w/o create-file if - drop false - else - journal-file-id ! - \ Write journal header (first block) - clear-block - JOURNAL-MAGIC block-buffer ! - 1 block-buffer 4 + ! \ version - 0 block-buffer 24 + ! \ journal head - 0 block-buffer 32 + ! \ checkpoint - 0 block-buffer 40 + ! \ entry count - BLOCK-SIZE block-buffer 48 + ! \ file size - block-buffer BLOCK-SIZE journal-file-id @ write-file if - drop false - else - drop - 0 journal-sequence ! - 0 journal-head ! - 0 journal-checkpoint ! - 0 journal-entry-count ! - BLOCK-SIZE journal-file-size ! - true - then - then ; - -\ Close journal file -: close-journal ( -- ) - journal-file-id @ close-file drop ; - -\ Flush journal to disk (ensure durability) -: flush-journal ( -- ) - journal-file-id @ flush-file drop ; - -\ ============================================================ -\ Journal Entry Creation -\ ============================================================ - -\ Clear entry buffer -: clear-entry ( -- ) - entry-buffer MAX-ENTRY-SIZE 0 fill ; - -\ Initialize entry header -: init-entry ( op-type affected-block -- ) - clear-entry - \ Increment sequence - 1 journal-sequence +! - journal-sequence @ entry-buffer entry-sequence ! - now-microseconds entry-buffer entry-timestamp ! - swap entry-buffer entry-op-type w! - 0 entry-buffer entry-flags w! - entry-buffer entry-affected ! - 0 entry-buffer entry-forward-len ! - 0 entry-buffer entry-inverse-len ! - 0 entry-buffer entry-provenance-len ! ; - -\ Set forward payload -: set-forward-payload ( addr len -- ) - dup entry-buffer entry-forward-len ! - entry-buffer entry-forward swap move ; - -\ Set inverse payload -: set-inverse-payload ( addr len -- ) - dup entry-buffer entry-inverse-len ! - entry-buffer entry-inverse swap move ; - -\ Set provenance payload -: set-provenance-payload ( addr len -- ) - dup entry-buffer entry-provenance-len ! - entry-buffer entry-provenance swap move ; - -\ Compute entry length -: compute-entry-len ( -- len ) - ENTRY-HEADER-SIZE - entry-buffer entry-forward-len @ + - entry-buffer entry-inverse-len @ + - entry-buffer entry-provenance-len @ + ; - -\ Finalize entry (compute checksum and length) -: finalize-entry ( -- ) - compute-entry-len dup entry-buffer entry-len ! - \ Compute checksum over entire entry except checksum field - entry-buffer swap crc32c - entry-buffer entry-checksum ! ; - -\ ============================================================ -\ Journal Writing -\ ============================================================ - -\ Append entry to journal -: append-entry ( -- flag ) - finalize-entry - \ Seek to end of journal - journal-file-size @ journal-file-id @ reposition-file if - drop false exit - then - drop - \ Write entry - entry-buffer entry-buffer entry-len @ - journal-file-id @ write-file if - drop false exit - then - drop - \ Update file size - entry-buffer entry-len @ journal-file-size +! - \ Update entry count - 1 journal-entry-count +! - true ; - -\ Mark entry as committed -: commit-entry ( -- ) - EFLAG-COMMITTED entry-buffer entry-flags w@ or - entry-buffer entry-flags w! - journal-sequence @ journal-head ! ; - -\ Write checkpoint entry -: write-checkpoint ( -- flag ) - OP-CHECKPOINT 0 init-entry - EFLAG-CHECKPOINT entry-buffer entry-flags w@ or - entry-buffer entry-flags w! - append-entry if - commit-entry - journal-head @ journal-checkpoint ! - true - else - false - then ; - -\ ============================================================ -\ Journal Reading -\ ============================================================ - -\ Read entry at offset -: read-entry-at ( offset -- flag ) - journal-file-id @ reposition-file if - drop false exit - then - drop - \ Read header first - entry-buffer ENTRY-HEADER-SIZE journal-file-id @ read-file if - drop false exit - then - ENTRY-HEADER-SIZE <> if false exit then - \ Read rest of entry - entry-buffer entry-len @ ENTRY-HEADER-SIZE - dup 0> if - entry-buffer ENTRY-HEADER-SIZE + swap - journal-file-id @ read-file if - drop false exit - then - drop - else - drop - then - true ; - -\ Validate entry checksum -: validate-entry ( -- flag ) - entry-buffer entry-checksum @ - entry-buffer entry-buffer entry-len @ crc32c - = ; - -\ ============================================================ -\ Journal Replay (Crash Recovery) -\ ============================================================ - -\ Apply a single journal entry by replaying its forward payload -: replay-entry ( -- flag ) - entry-buffer entry-op-type w@ - case - OP-DOC-INSERT of - \ Recreate document block from forward payload - entry-buffer entry-affected @ - TYPE-DOCUMENT over init-block-header - entry-buffer entry-forward - entry-buffer entry-forward-len @ - dup 0> if - set-block-payload - block-buffer write-block - else - 2drop drop true \ Empty payload, skip - then - endof - OP-DOC-UPDATE of - \ Overwrite block with forward payload (new data) - entry-buffer entry-affected @ - block-buffer read-block if - entry-buffer entry-forward - entry-buffer entry-forward-len @ - dup 0> if - set-block-payload - entry-buffer entry-affected @ - block-buffer write-block - else - 2drop true - then - else - false - then - endof - OP-DOC-DELETE of - \ Mark block as deleted - entry-buffer entry-affected @ free-block true - endof - OP-SCHEMA-CREATE of - \ Recreate schema block from forward payload - entry-buffer entry-affected @ - TYPE-SCHEMA over init-block-header - entry-buffer entry-forward - entry-buffer entry-forward-len @ - dup 0> if - set-block-payload - block-buffer write-block - else - 2drop drop true - then - endof - OP-COLLECTION-CREATE of - \ Collection metadata is in-memory only; skip block replay - true - endof - OP-CHECKPOINT of - true \ Checkpoints are markers, nothing to replay - endof - \ Default: skip unknown operations - true swap - endcase ; - -\ Replay uncommitted entries for crash recovery -: replay-journal ( -- ) - 0 >r \ r: replayed count - BLOCK-SIZE \ Start after journal header block - begin - dup journal-file-size @ < - while - dup read-entry-at if - validate-entry if - entry-buffer entry-flags w@ - dup EFLAG-COMMITTED and 0= swap - EFLAG-ROLLED-BACK and 0= and if - \ Entry not committed and not rolled back — replay it - ." Replaying entry seq=" entry-buffer entry-sequence @ . - ." op=" entry-buffer entry-op-type w@ .op-type cr - replay-entry if - \ Mark as committed after successful replay - commit-entry - r> 1+ >r - else - ." Warning: failed to replay entry seq=" - entry-buffer entry-sequence @ . cr - then - then - else - ." Warning: corrupt entry at offset " dup . cr - then - entry-buffer entry-len @ + - else - ." Error reading entry at offset " dup . cr - drop r> drop exit - then - repeat - drop - r> dup 0> if - ." Replay complete: " . ." entries recovered" cr - flush-db \ Ensure all replayed data is durable - else - drop ." No entries to replay" cr - then ; - -\ ============================================================ -\ Canonical Rendering -\ ============================================================ - -\ Render operation type -: .op-type ( op -- ) - case - OP-DOC-INSERT of ." DOC_INSERT" endof - OP-DOC-UPDATE of ." DOC_UPDATE" endof - OP-DOC-DELETE of ." DOC_DELETE" endof - OP-EDGE-INSERT of ." EDGE_INSERT" endof - OP-EDGE-DELETE of ." EDGE_DELETE" endof - OP-COLLECTION-CREATE of ." COLLECTION_CREATE" endof - OP-COLLECTION-DROP of ." COLLECTION_DROP" endof - OP-CHECKPOINT of ." CHECKPOINT" endof - OP-IRREVERSIBLE of ." IRREVERSIBLE" endof - ." UNKNOWN" - endcase ; - -\ Render entry flags -: .entry-flags ( flags -- ) - ." [" - dup EFLAG-COMMITTED and if ." COMMITTED " then - dup EFLAG-ROLLED-BACK and if ." ROLLED_BACK " then - dup EFLAG-CHECKPOINT and if ." CHECKPOINT " then - dup EFLAG-COMPRESSED and if ." COMPRESSED " then - dup EFLAG-IRREVERSIBLE and if ." IRREVERSIBLE " then - drop - ." ]" ; - -\ Render journal entry header -: .entry-header ( addr -- ) - ." JOURNAL seq=" dup entry-sequence @ . - ." op=" dup entry-op-type w@ .op-type - ." timestamp=" dup entry-timestamp @ . cr - ." affected_block=" dup entry-affected @ . - ." flags=" entry-flags w@ .entry-flags cr ; - -\ ============================================================ -\ High-Level Journal Operations -\ ============================================================ - -\ Begin journaled operation -: begin-journal-op ( op-type block-id -- ) - init-entry ; - -\ End journaled operation (append and commit) -: end-journal-op ( -- flag ) - append-entry if - commit-entry - true - else - false - then ; - -\ Rollback uncommitted operation -: rollback-journal-op ( -- ) - EFLAG-ROLLED-BACK entry-buffer entry-flags w@ or - entry-buffer entry-flags w! ; - -\ Write-ahead log pattern: journal → flush → write block → commit → flush -\ Caller must have already called begin-journal-op and set payloads. -\ ( block-id addr -- flag ) -: wal-write-block - \ Step 1: Append journal entry to disk - append-entry 0= if 2drop false exit then - \ Step 2: Flush journal for durability (WAL guarantee) - flush-journal - \ Step 3: Write the actual data block - write-block 0= if - rollback-journal-op false exit - then - \ Step 4: Mark journal entry as committed - commit-entry - \ Step 5: Flush commit marker - flush-journal - true ; - -\ ============================================================ -\ Initialization -\ ============================================================ - -: init-journal ( -- ) - 0 journal-sequence ! - 0 journal-head ! - 0 journal-checkpoint ! - 0 journal-entry-count ! - BLOCK-SIZE journal-file-size ! - clear-entry ; - -init-journal diff --git a/lithoglyph/core-forth/src/lithoglyph-model.fs b/lithoglyph/core-forth/src/lithoglyph-model.fs deleted file mode 100644 index cd9b85b1..00000000 --- a/lithoglyph/core-forth/src/lithoglyph-model.fs +++ /dev/null @@ -1,331 +0,0 @@ -\ SPDX-License-Identifier: MPL-2.0 -\ Form.Model - Multi-model logical layer -\ -\ Documents, edges, collections, and schemas. -\ All operations go through the journal. -\ -\ Part of Lithoglyph: Stone-carved data for the ages. - -require lithoglyph-journal.fs - -\ ============================================================ -\ Collection Management -\ ============================================================ - -\ Collection metadata structure -\ - name (up to 64 bytes) -\ - type (document or edge) -\ - schema_id (0 if no schema) -\ - document_count -\ - created_at -\ - modified_at - -64 constant MAX-COLLECTION-NAME - -\ Collection types -0 constant CTYPE-DOCUMENT -1 constant CTYPE-EDGE - -\ In-memory collection registry (simple linked list for PoC) -variable collection-list \ Head of collection list -0 collection-list ! - -\ Collection node structure -struct - cell% field coll-next \ Next collection in list - cell% field coll-block-id \ Block ID of collection metadata - cell% field coll-type \ Document or edge - cell% field coll-schema-id \ Schema block ID (0 if none) - cell% field coll-doc-count \ Number of documents - MAX-COLLECTION-NAME 1+ char% field coll-name -end-struct collection% - -\ Allocate new collection node -: alloc-collection ( -- addr ) - collection% %allot ; - -\ Find collection by name -: find-collection ( addr len -- coll-addr | 0 ) - collection-list @ - begin - dup 0<> - while - >r - 2dup r@ coll-name count compare 0= if - 2drop r> exit - then - r> coll-next @ - repeat - 2drop 0 ; - -\ Create new collection -: create-collection ( name-addr name-len type -- coll-addr | 0 ) - \ Check if already exists - 2 pick 2 pick find-collection if - drop 2drop 0 exit - then - \ Allocate block for collection metadata - alloc-block-id - \ Create journal entry - OP-COLLECTION-CREATE over begin-journal-op - \ Allocate collection node - alloc-collection >r - r@ coll-block-id ! - r@ coll-type ! - dup r@ coll-name c! - r@ coll-name 1+ swap move - 0 r@ coll-schema-id ! - 0 r@ coll-doc-count ! - \ Link into list - collection-list @ r@ coll-next ! - r@ collection-list ! - \ Commit journal entry - end-journal-op drop - r> ; - -\ Drop collection -: drop-collection ( name-addr name-len -- flag ) - find-collection dup 0= if exit then - \ Create journal entry - OP-COLLECTION-DROP over coll-block-id @ begin-journal-op - \ TODO: Actually remove from list and free blocks - end-journal-op ; - -\ ============================================================ -\ Document Operations -\ ============================================================ - -\ Document ID structure (16 bytes: prefix + counter) -16 constant DOC-ID-SIZE - -variable doc-counter -0 doc-counter ! - -\ Generate new document ID -create doc-id-buffer DOC-ID-SIZE allot -: gen-doc-id ( -- addr ) - \ Simple counter-based ID for PoC - 1 doc-counter +! - doc-id-buffer DOC-ID-SIZE 0 fill - s" doc_" doc-id-buffer swap move - doc-counter @ doc-id-buffer 4 + ! - doc-id-buffer ; - -\ Insert document into collection -\ WAL pattern: journal → flush → write block → commit → flush -: insert-document ( coll-addr payload-addr payload-len -- doc-id | 0 ) - \ Validate collection exists - over 0= if 2drop drop 0 exit then - \ Allocate block for document - alloc-block-id >r - \ Initialize document block - TYPE-DOCUMENT r@ init-block-header - \ Link document to collection via prev_block_id - 3 pick coll-block-id @ block-buffer block-prev ! - \ Keep a copy of payload for forward journal payload - 2dup - \ Set payload into block buffer (consumes top copy) - set-block-payload - \ Begin journal entry - OP-DOC-INSERT r@ begin-journal-op - \ Set forward payload (the document data being inserted) - set-forward-payload - \ WAL: journal first, then write block - r@ block-buffer wal-write-block if - \ Increment document count - 1 swap coll-doc-count +! - gen-doc-id - else - drop 0 - then - r> drop ; - -\ Update document -\ WAL pattern with inverse payload for undo capability -: update-document ( doc-block-id field-addr field-len value-addr value-len -- flag ) - \ Read existing block - 4 pick block-buffer read-block 0= if 2drop 2drop drop false exit then - \ Begin journal entry - OP-DOC-UPDATE 4 pick begin-journal-op - \ Capture inverse payload (old block data, before overwrite) - block-buffer block-payload block-buffer block-payload-len l@ - set-inverse-payload - \ Drop field name (field-level update not yet implemented) - \ Stack: doc-block-id field-addr field-len value-addr value-len - 2swap 2drop - \ Stack: doc-block-id value-addr value-len - \ Set forward payload (new data) and update block buffer - 2dup set-forward-payload - set-block-payload - \ Stack: doc-block-id - \ WAL: journal first, then write block - dup block-buffer wal-write-block - nip ; - -\ Delete document -\ WAL pattern with inverse payload for undo capability -: delete-document ( coll-addr doc-block-id -- flag ) - \ Read block to capture inverse payload - dup block-buffer read-block 0= if 2drop false exit then - \ Begin journal entry - OP-DOC-DELETE over begin-journal-op - \ Save inverse payload (full document content for undo) - block-buffer block-payload block-buffer block-payload-len l@ - set-inverse-payload - \ WAL: Journal first - append-entry 0= if 2drop false exit then - flush-journal - \ Now safe to mark block as deleted - free-block - \ Commit journal entry - commit-entry - flush-journal - \ Update collection count - -1 swap coll-doc-count +! - true ; - -\ ============================================================ -\ Edge Operations -\ ============================================================ - -\ Edge structure in payload: -\ - from_collection (64 bytes) -\ - from_id (16 bytes) -\ - to_collection (64 bytes) -\ - to_id (16 bytes) -\ - edge_type (64 bytes) -\ - properties (CBOR, variable) - -224 constant EDGE-HEADER-SIZE - -\ Create edge buffer -create edge-buffer PAYLOAD-SIZE allot - -\ Build edge payload -: build-edge ( from-coll from-id to-coll to-id edge-type props-addr props-len -- addr len ) - edge-buffer PAYLOAD-SIZE 0 fill - \ Copy edge type (at offset 160) - >r >r - edge-buffer 160 + 64 0 fill - over edge-buffer 160 + swap move drop - \ Copy to_id (at offset 144) - edge-buffer 144 + 16 0 fill - swap edge-buffer 144 + swap move drop - \ Copy to_collection (at offset 80) - edge-buffer 80 + 64 0 fill - over edge-buffer 80 + swap move drop - \ Copy from_id (at offset 64) - edge-buffer 64 + 16 0 fill - swap edge-buffer 64 + swap move drop - \ Copy from_collection (at offset 0) - edge-buffer 64 0 fill - over edge-buffer swap move drop - \ Copy properties - r> r> - edge-buffer EDGE-HEADER-SIZE + swap move - edge-buffer EDGE-HEADER-SIZE + ; - -\ Insert edge -: insert-edge ( edge-coll from-coll from-id to-coll to-id type props-addr props-len -- flag ) - \ Build edge payload - build-edge - \ Insert as document in edge collection - rot insert-document - 0<> ; - -\ ============================================================ -\ Schema Operations -\ ============================================================ - -\ Schema structure: -\ - version (4 bytes) -\ - field_count (4 bytes) -\ - fields (variable: name + type + constraints) - -\ Create schema -\ WAL pattern: journal → flush → write block → commit -: create-schema ( coll-addr schema-payload-addr schema-len -- flag ) - \ Allocate block for schema - alloc-block-id >r - TYPE-SCHEMA r@ init-block-header - \ Keep copy for forward payload - 2dup set-forward-payload - set-block-payload - \ Begin journal entry - OP-SCHEMA-CREATE r@ begin-journal-op - \ WAL: journal first, then write block - r@ block-buffer wal-write-block if - \ Link schema to collection - r> swap coll-schema-id ! - true - else - r> drop drop - false - then ; - -\ ============================================================ -\ Query Support (Minimal) -\ ============================================================ - -\ Callback type for iteration -\ ( block-addr -- continue? ) - -\ Variables for foreach-document iteration -variable foreach-coll-block -variable foreach-callback - -\ Iterate all documents in collection -\ Scans all blocks and filters by type=DOCUMENT, matching collection via prev_block_id. -\ The callback receives block-buffer address and returns true to continue, false to stop. -: foreach-document ( coll-addr xt -- ) - foreach-callback ! - coll-block-id @ foreach-coll-block ! - next-block-id @ 1 ?do - i block-buffer read-block if - block-buffer block-type w@ TYPE-DOCUMENT = if - block-buffer block-flags l@ FLAG-DELETED and 0= if - block-buffer block-prev @ foreach-coll-block @ = if - block-buffer foreach-callback @ execute 0= if - unloop exit - then - then - then - then - then - loop ; - -\ ============================================================ -\ Canonical Rendering -\ ============================================================ - -\ Render collection info -: .collection ( coll-addr -- ) - ." Collection: " dup coll-name count type cr - ." type: " dup coll-type @ case - CTYPE-DOCUMENT of ." document" endof - CTYPE-EDGE of ." edge" endof - endcase cr - ." documents: " dup coll-doc-count @ . cr - ." schema_id: " coll-schema-id @ . cr ; - -\ List all collections -: .collections ( -- ) - ." Collections:" cr - collection-list @ - begin - dup 0<> - while - ." " dup coll-name count type cr - coll-next @ - repeat - drop ; - -\ ============================================================ -\ Initialization -\ ============================================================ - -: init-model ( -- ) - 0 collection-list ! - 0 doc-counter ! ; - -init-model diff --git a/lithoglyph/core-forth/test/test-blocks.fs b/lithoglyph/core-forth/test/test-blocks.fs deleted file mode 100644 index 29485e55..00000000 --- a/lithoglyph/core-forth/test/test-blocks.fs +++ /dev/null @@ -1,130 +0,0 @@ -\ SPDX-License-Identifier: MPL-2.0 -\ Test harness for Form.Blocks -\ -\ Part of Lithoglyph: Stone-carved data for the ages. - -include ../src/lithoglyph-blocks.fs - -\ ============================================================ -\ Test Utilities -\ ============================================================ - -variable test-count -variable pass-count -variable fail-count - -: test-start ( -- ) - 0 test-count ! - 0 pass-count ! - 0 fail-count ! - cr ." ========================================" cr - ." Form.Blocks Test Suite" cr - ." ========================================" cr ; - -: test ( c-addr u -- ) - 1 test-count +! - cr ." TEST: " type ." ... " ; - -: pass ( -- ) - 1 pass-count +! - ." PASS" ; - -: fail ( -- ) - 1 fail-count +! - ." FAIL" ; - -: assert= ( n1 n2 -- ) - = if pass else fail then ; - -: assert-true ( flag -- ) - if pass else fail then ; - -: assert-false ( flag -- ) - if fail else pass then ; - -: test-summary ( -- ) - cr ." ========================================" cr - ." Results: " - pass-count @ . ." passed, " - fail-count @ . ." failed, " - test-count @ . ." total" cr - ." ========================================" cr ; - -\ ============================================================ -\ Actual Tests -\ ============================================================ - -test-start - -\ Test constants -s" BLOCK-SIZE is 4096" test -BLOCK-SIZE 4096 assert= - -s" HEADER-SIZE is 64" test -HEADER-SIZE 64 assert= - -s" PAYLOAD-SIZE is 4032" test -PAYLOAD-SIZE 4032 assert= - -s" TYPE-DOCUMENT is $0011" test -TYPE-DOCUMENT $0011 assert= - -\ Test block buffer allocation -s" block-buffer is allocated" test -block-buffer 0<> assert-true - -\ Test clear-block -s" clear-block works" test -clear-block -block-buffer @ 0 assert= - -\ Test CRC32C initialization -s" CRC32C table initialized (entry 1 non-zero)" test -crc32c-table 1 cells + @ 0<> assert-true - -\ Test CRC32C calculation using block-buffer -s" CRC32C of buffer data is computed" test -\ Debug: show block-buffer address -\ ." block-buffer addr = " block-buffer . cr -\ Write some bytes to block-buffer payload area and compute CRC -$41 block-buffer block-payload c! -$42 block-buffer block-payload 1+ c! -$43 block-buffer block-payload 2 + c! -$44 block-buffer block-payload 3 + c! -block-buffer block-payload 4 crc32c 0<> assert-true - -\ Test block header initialization -s" init-block-header sets magic" test -TYPE-DOCUMENT 1 init-block-header -block-buffer blk-magic l@ BLOCK-MAGIC assert= - -s" init-block-header sets version" test -block-buffer block-version w@ 1 assert= - -s" init-block-header sets type" test -block-buffer block-type w@ TYPE-DOCUMENT assert= - -s" init-block-header sets block-id" test -block-buffer block-id @ 1 assert= - -\ Test block validation -s" valid-magic? works" test -block-buffer valid-magic? assert-true - -s" valid-type? works" test -block-buffer valid-type? assert-true - -s" valid-payload-len? works" test -block-buffer valid-payload-len? assert-true - -\ Test payload setting -s" set-block-payload works" test -s" test payload" drop 12 set-block-payload -block-buffer block-payload-len l@ 12 assert= - -s" checksum is computed" test -block-buffer block-checksum l@ 0<> assert-true - -test-summary - -bye diff --git a/lithoglyph/core-zig/README.adoc b/lithoglyph/core-zig/README.adoc deleted file mode 100644 index 4ce27400..00000000 --- a/lithoglyph/core-zig/README.adoc +++ /dev/null @@ -1,37 +0,0 @@ -= Form.Bridge (Zig) -:toc: - -The ABI boundary layer, implemented in Zig. - -== Purpose - -* Stable C ABI for runtimes -* Safety governor -* Marshalling (no business logic duplication) - -== API Surface - -[source,c] ----- -// Lifecycle -lith_db_open(path, opts_bytes, opts_len) -> (db*, status, err_blob) -lith_db_close(db*) -> status - -// Transactions -lith_txn_begin(db*, mode) -> (txn*, status, err_blob) -lith_txn_commit(txn*) -> (status, err_blob) -lith_txn_abort(txn*) -> status - -// Operations -lith_apply(txn*, op_bytes, op_len) -> (result_blob, provenance_blob, status, err_blob) - -// Introspection -lith_render_block(db*, block_id, render_opts) -> (text_blob, status, err_blob) -lith_render_journal(db*, since, render_opts) -> (text_blob, status, err_blob) -lith_introspect_schema(db*) -> (schema_blob, status, err_blob) -lith_introspect_constraints(db*) -> (constraints_blob, status, err_blob) ----- - -== Status - -Implementation pending specification completion. diff --git a/lithoglyph/core-zig/build.zig b/lithoglyph/core-zig/build.zig deleted file mode 100644 index d4366c15..00000000 --- a/lithoglyph/core-zig/build.zig +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Form.Bridge - Build Configuration (Zig 0.15.2+) - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Main static library - const lib = b.addLibrary(.{ - .name = "lith_bridge", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - }), - .linkage = .static, - }); - - // Also build shared library for FFI - const shared_lib = b.addLibrary(.{ - .name = "lith_bridge", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - }), - .linkage = .dynamic, - }); - - // Install artifacts - b.installArtifact(lib); - b.installArtifact(shared_lib); - - // Unit tests for bridge - const bridge_tests = b.addTest(.{ - .name = "bridge-tests", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - }), - }); - - const run_bridge_tests = b.addRunArtifact(bridge_tests); - - // Unit tests for blocks - const blocks_tests = b.addTest(.{ - .name = "blocks-tests", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/blocks.zig"), - .target = target, - .optimize = optimize, - }), - }); - - // Unit tests for crypto - const _crypto_tests = b.addTest(.{ - .name = "crypto-tests", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/crypto_test.zig"), - .target = target, - .optimize = optimize, - }), - }); - - const run_blocks_tests = b.addRunArtifact(blocks_tests); - const run_crypto_tests = b.addRunArtifact(crypto_tests); - - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_bridge_tests.step); - test_step.dependOn(&run_blocks_tests.step); - test_step.dependOn(&run_crypto_tests.step); -} diff --git a/lithoglyph/core-zig/src/blocks.zig b/lithoglyph/core-zig/src/blocks.zig deleted file mode 100644 index 15eb81af..00000000 --- a/lithoglyph/core-zig/src/blocks.zig +++ /dev/null @@ -1,633 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph Block Storage - Forth-Compatible Implementation -// -// Implements the block format specified in core-forth/src/lithoglyph-blocks.fs -// All blocks are 4KiB with 64-byte headers and CRC32C checksums. -// -// This is the truth core - stone-carved data for the ages. - -const std = @import("std"); -const builtin = @import("builtin"); -const crypto = @import("crypto.zig"); - -// ============================================================ -// Constants (must match Forth specification) -// ============================================================ - -pub const BLOCK_SIZE: usize = 4096; // 4 KiB blocks -pub const HEADER_SIZE: usize = 64; // 64-byte fixed header -pub const PAYLOAD_SIZE: usize = BLOCK_SIZE - HEADER_SIZE; // 4032 bytes - -// Magic bytes: "LGH\0" = 0x4C474800 (Lithoglyph) -pub const BLOCK_MAGIC: u32 = 0x4C474800; -pub const BLOCK_VERSION: u16 = 1; - -// Block Types (must match Forth definitions) -pub const BlockType = enum(u16) { - free = 0x0000, - superblock = 0x0001, - collection_meta = 0x0010, - document = 0x0011, - document_overflow = 0x0012, - edge_meta = 0x0020, - edge = 0x0021, - index_root = 0x0030, - index_internal = 0x0031, - index_leaf = 0x0032, - journal_segment = 0x0040, - schema = 0x0050, - constraint = 0x0051, - migration = 0x0060, -}; - -// Block Flags (bitmask) -pub const BlockFlags = packed struct { - compressed: bool = false, - encrypted: bool = false, - chained: bool = false, - deleted: bool = false, - _reserved: u4 = 0, -}; - -// ============================================================ -// Block Header Structure (64 bytes, matching Forth layout) -// ============================================================ -// -// Offset Size Field -// 0 4 magic -// 4 2 version -// 6 2 block_type -// 8 8 block_id -// 16 8 sequence -// 24 8 created_at -// 32 8 modified_at -// 40 4 payload_len -// 44 4 checksum -// 48 8 prev_block_id -// 56 4 flags -// 60 4 reserved - -pub const BlockHeader = extern struct { - magic: u32 align(1), - version: u16 align(1), - block_type: u16 align(1), - block_id: u64 align(1), - sequence: u64 align(1), - created_at: u64 align(1), - modified_at: u64 align(1), - payload_len: u32 align(1), - checksum: u32 align(1), - prev_block_id: u64 align(1), - flags: u32 align(1), - reserved: u32 align(1), - - comptime { - // Verify at compile time that struct is exactly 64 bytes - if (@sizeOf(BlockHeader) != HEADER_SIZE) { - @compileError("BlockHeader must be exactly 64 bytes"); - } - } - - /// Initialize a new block header - pub fn init(block_type: BlockType, block_id: u64, sequence: u64) BlockHeader { - const now = @as(u64, @intCast(std.time.milliTimestamp())); - return .{ - .magic = BLOCK_MAGIC, - .version = BLOCK_VERSION, - .block_type = @intFromEnum(block_type), - .block_id = block_id, - .sequence = sequence, - .created_at = now, - .modified_at = now, - .payload_len = 0, - .checksum = 0, - .prev_block_id = 0, - .flags = 0, - .reserved = 0, - }; - } - - /// Validate block header - pub fn validate(self: *const BlockHeader) !void { - if (self.magic != BLOCK_MAGIC) { - return error.InvalidMagic; - } - if (self.version != BLOCK_VERSION) { - return error.UnsupportedVersion; - } - if (self.payload_len > PAYLOAD_SIZE) { - return error.PayloadTooLarge; - } - } - - /// Convert to native endianness (blocks are stored little-endian) - pub fn toNative(self: *BlockHeader) void { - if (builtin.cpu.arch.endian() != .little) { - self.magic = @byteSwap(self.magic); - self.version = @byteSwap(self.version); - self.block_type = @byteSwap(self.block_type); - self.block_id = @byteSwap(self.block_id); - self.sequence = @byteSwap(self.sequence); - self.created_at = @byteSwap(self.created_at); - self.modified_at = @byteSwap(self.modified_at); - self.payload_len = @byteSwap(self.payload_len); - self.checksum = @byteSwap(self.checksum); - self.prev_block_id = @byteSwap(self.prev_block_id); - self.flags = @byteSwap(self.flags); - self.reserved = @byteSwap(self.reserved); - } - } - - /// Convert from native endianness to little-endian for storage - pub fn toLittleEndian(self: *BlockHeader) void { - // Same as toNative - converts between current and little-endian - self.toNative(); - } -}; - -// ============================================================ -// Complete Block (Header + Payload) -// ============================================================ - -pub const Block = struct { - header: BlockHeader, - payload: [PAYLOAD_SIZE]u8, - - /// Initialize a new block - pub fn init(block_type: BlockType, block_id: u64, sequence: u64) Block { - var block = Block{ - .header = BlockHeader.init(block_type, block_id, sequence), - .payload = undefined, - }; - @memset(&block.payload, 0); - return block; - } - - /// Set payload data and update header - pub fn setPayload(self: *Block, data: []const u8) !void { - if (data.len > PAYLOAD_SIZE) { - return error.PayloadTooLarge; - } - - // Copy payload - @memcpy(self.payload[0..data.len], data); - - // Zero remaining space - if (data.len < PAYLOAD_SIZE) { - @memset(self.payload[data.len..], 0); - } - - // Update header - self.header.payload_len = @intCast(data.len); - self.header.modified_at = @intCast(std.time.milliTimestamp()); - - // Calculate checksum - self.header.checksum = crc32c(&self.payload, self.header.payload_len); - } - - /// Get payload as slice - pub fn getPayload(self: *const Block) []const u8 { - return self.payload[0..self.header.payload_len]; - } - - /// Validate block (header + checksum) - pub fn validate(self: *const Block) !void { - try self.header.validate(); - - // Verify checksum - const computed = crc32c(&self.payload, self.header.payload_len); - if (computed != self.header.checksum) { - return error.ChecksumMismatch; - } - } - - /// Write block to bytes (for disk storage) - pub fn toBytes(self: *const Block) [BLOCK_SIZE]u8 { - var bytes: [BLOCK_SIZE]u8 = undefined; - - // Copy header - const header_bytes = std.mem.asBytes(&self.header); - @memcpy(bytes[0..HEADER_SIZE], header_bytes); - - // Copy payload - @memcpy(bytes[HEADER_SIZE..], &self.payload); - - return bytes; - } - - /// Read block from bytes - pub fn fromBytes(bytes: *const [BLOCK_SIZE]u8) !Block { - var block: Block = undefined; - - // Read header - const header_bytes = bytes[0..HEADER_SIZE]; - block.header = @bitCast(header_bytes.*); - block.header.toNative(); - - // Read payload - @memcpy(&block.payload, bytes[HEADER_SIZE..]); - - // Validate - try block.validate(); - - return block; - } - - /// Encrypt block payload using AES-256-GCM - /// Requires 32-byte encryption key - pub fn encrypt(self: *Block, key: [crypto.AES256_KEY_SIZE]u8) !void { - try crypto.encryptBlockPayload(self, key); - } - - /// Decrypt block payload using AES-256-GCM - /// Requires 32-byte encryption key - pub fn decrypt(self: *Block, key: [crypto.AES256_KEY_SIZE]u8) !void { - try crypto.decryptBlockPayload(self, key); - } - - /// Check if block payload is encrypted - pub fn isEncrypted(self: *const Block) bool { - return (self.header.flags & 0x02) != 0; // bit 1 = encrypted in BlockFlags - } - - /// Derive encryption key from master key - pub fn deriveKey(master_key: []const u8, block_type: BlockType, block_id: u64) ![crypto.AES256_KEY_SIZE]u8 { - return crypto.deriveBlockKey(master_key, block_type, block_id); - } -}; - -// ============================================================ -// CRC32C Implementation (Castagnoli polynomial: 0x1EDC6F41) -// ============================================================ -// This matches the Forth implementation for bit-exact compatibility - -const CRC32C_TABLE = blk: { - @setEvalBranchQuota(3000); // Allow compile-time table generation - var table: [256]u32 = undefined; - for (&table, 0..) |*entry, i| { - var crc: u32 = @intCast(i); - var j: usize = 0; - while (j < 8) : (j += 1) { - if (crc & 1 != 0) { - crc = (crc >> 1) ^ 0x82F63B78; - } else { - crc >>= 1; - } - } - entry.* = crc; - } - break :blk table; -}; - -/// Calculate CRC32C checksum (Castagnoli) -pub fn crc32c(data: []const u8, len: u32) u32 { - var crc: u32 = 0xFFFFFFFF; - - var i: usize = 0; - while (i < len) : (i += 1) { - const index = @as(u8, @truncate((crc ^ data[i]) & 0xFF)); - crc = (crc >> 8) ^ CRC32C_TABLE[index]; - } - - return crc ^ 0xFFFFFFFF; -} - -// ============================================================ -// Superblock (Block ID 0) -// ============================================================ - -pub const Superblock = extern struct { - version: u32 align(1), - block_count: u64 align(1), - free_list_head: u64 align(1), - journal_head: u64 align(1), - journal_tail: u64 align(1), - root_collection_id: u64 align(1), - flags: u32 align(1), - created_at: u64 align(1), - last_checkpoint: u64 align(1), - reserved: [3968]u8 align(1), // Pad to payload size - - pub fn init() Superblock { - const now = @as(u64, @intCast(std.time.milliTimestamp())); - return .{ - .version = 1, - .block_count = 1, // Just the superblock - .free_list_head = 0, - .journal_head = 0, - .journal_tail = 0, - .root_collection_id = 0, - .flags = 0, - .created_at = now, - .last_checkpoint = now, - .reserved = undefined, - }; - } - - pub fn toBlock(self: *const Superblock) !Block { - var block = Block.init(.superblock, 0, 0); - const bytes = std.mem.asBytes(self); - try block.setPayload(bytes); - return block; - } - - pub fn fromBlock(block: *const Block) !Superblock { - if (block.header.block_type != @intFromEnum(BlockType.superblock)) { - return error.NotSuperblock; - } - const bytes = block.getPayload(); - if (bytes.len < @sizeOf(Superblock)) { - return error.InvalidSuperblock; - } - return @bitCast(bytes[0..@sizeOf(Superblock)].*); - } -}; - -// ============================================================ -// Block Storage Manager -// ============================================================ - -pub const BlockStorage = struct { - allocator: std.mem.Allocator, - file: std.fs.File, - superblock: Superblock, - path: []const u8, - is_open: bool, - - /// Open or create block storage - pub fn open(allocator: std.mem.Allocator, path: []const u8) !*BlockStorage { - const storage = try allocator.create(BlockStorage); - errdefer allocator.destroy(storage); - - // Try to open existing file - const file = std.fs.cwd().openFile(path, .{ .mode = .read_write }) catch |err| { - if (err == error.FileNotFound) { - // Create new database - const new_file = try std.fs.cwd().createFile(path, .{ .read = true }); - const sb = Superblock.init(); - const sb_block = try sb.toBlock(); - const sb_bytes = sb_block.toBytes(); - try new_file.writeAll(&sb_bytes); - try new_file.sync(); - - storage.* = .{ - .allocator = allocator, - .file = new_file, - .superblock = sb, - .path = try allocator.dupe(u8, path), - .is_open = true, - }; - return storage; - } - return err; - }; - - // Read existing superblock - var sb_bytes: [BLOCK_SIZE]u8 = undefined; - const n = try file.read(&sb_bytes); - if (n < BLOCK_SIZE) { - return error.InvalidDatabase; - } - - const sb_block = try Block.fromBytes(&sb_bytes); - const sb = try Superblock.fromBlock(&sb_block); - - storage.* = .{ - .allocator = allocator, - .file = file, - .superblock = sb, - .path = try allocator.dupe(u8, path), - .is_open = true, - }; - - return storage; - } - - /// Close block storage - pub fn close(self: *BlockStorage) void { - if (self.is_open) { - self.file.close(); - self.allocator.free(self.path); - self.is_open = false; - } - } - - pub fn deinit(self: *BlockStorage) void { - self.close(); - self.allocator.destroy(self); - } - - /// Read a block by ID - pub fn readBlock(self: *BlockStorage, block_id: u64) !Block { - const offset = block_id * BLOCK_SIZE; - try self.file.seekTo(offset); - - var bytes: [BLOCK_SIZE]u8 = undefined; - const n = try self.file.read(&bytes); - if (n < BLOCK_SIZE) { - return error.InvalidBlock; - } - - return try Block.fromBytes(&bytes); - } - - /// Write a block by ID - pub fn writeBlock(self: *BlockStorage, block_id: u64, block: *const Block) !void { - const offset = block_id * BLOCK_SIZE; - try self.file.seekTo(offset); - - var bytes = block.toBytes(); - try self.file.writeAll(&bytes); - try self.file.sync(); - } - - /// Allocate a new block (writes to disk immediately) - pub fn allocateBlock(self: *BlockStorage, block_type: BlockType) !u64 { - // TODO: Use free list for production - // For now, just append to end - const new_id = self.superblock.block_count; - self.superblock.block_count += 1; - - // Update superblock on disk - const sb_block = try self.superblock.toBlock(); - try self.writeBlock(0, &sb_block); - - // Initialize new block - var block = Block.init(block_type, new_id, self.superblock.journal_head + 1); - try self.writeBlock(new_id, &block); - - return new_id; - } - - /// Reserve a block ID without writing to disk (for transaction buffering) - pub fn reserveBlockId(self: *BlockStorage) u64 { - const new_id = self.superblock.block_count; - self.superblock.block_count += 1; - return new_id; - } - - /// Flush superblock to disk (after batch operations) - pub fn flushSuperblock(self: *BlockStorage) !void { - const sb_block = try self.superblock.toBlock(); - try self.writeBlock(0, &sb_block); - } - - /// Append to journal - pub fn appendJournal(self: *BlockStorage, entry_data: []const u8) !u64 { - const journal_id = try self.allocateBlock(.journal_segment); - var block = try self.readBlock(journal_id); - try block.setPayload(entry_data); - - // Link to previous journal entry - block.header.prev_block_id = self.superblock.journal_tail; - block.header.sequence = self.superblock.journal_head + 1; - - try self.writeBlock(journal_id, &block); - - // Update superblock journal pointers - if (self.superblock.journal_head == 0) { - self.superblock.journal_head = journal_id; - } - self.superblock.journal_tail = journal_id; - self.superblock.journal_head += 1; - - const sb_block = try self.superblock.toBlock(); - try self.writeBlock(0, &sb_block); - - return journal_id; - } - - /// Free a block (mark as free, add to free list) - pub fn freeBlock(self: *BlockStorage, block_id: u64) !void { - if (block_id == 0) { - return error.CannotFreeSuperblock; - } - - var block = try self.readBlock(block_id); - block.header.block_type = @intFromEnum(BlockType.free); - block.header.prev_block_id = self.superblock.free_list_head; - block.header.flags |= @as(u32, @as(u8, @bitCast(BlockFlags{ .deleted = true }))); - - try self.writeBlock(block_id, &block); - - // Update free list - self.superblock.free_list_head = block_id; - const sb_block = try self.superblock.toBlock(); - try self.writeBlock(0, &sb_block); - } -}; - -// ============================================================ -// Tests -// ============================================================ - -test "block header size" { - try std.testing.expectEqual(@as(usize, 64), @sizeOf(BlockHeader)); -} - -test "block size" { - try std.testing.expectEqual(@as(usize, 4096), @sizeOf(Block)); -} - -test "crc32c" { - const data = "hello world"; - const crc = crc32c(data, data.len); - // Verify against known CRC32C value - try std.testing.expect(crc != 0); -} - -test "block init and payload" { - var block = Block.init(.document, 1, 1); - const data = "test document data"; - try block.setPayload(data); - - try std.testing.expectEqual(data.len, block.header.payload_len); - try std.testing.expectEqualSlices(u8, data, block.getPayload()); - - // Validate checksum - try block.validate(); -} - -test "block serialization" { - var block = Block.init(.document, 42, 100); - try block.setPayload("test"); - - const bytes = block.toBytes(); - const decoded = try Block.fromBytes(&bytes); - - try std.testing.expectEqual(block.header.block_id, decoded.header.block_id); - try std.testing.expectEqual(block.header.sequence, decoded.header.sequence); - try std.testing.expectEqualSlices(u8, block.getPayload(), decoded.getPayload()); -} - -test "superblock roundtrip" { - var sb = Superblock.init(); - sb.block_count = 42; - sb.journal_head = 100; - - const block = try sb.toBlock(); - const decoded = try Superblock.fromBlock(&block); - - try std.testing.expectEqual(sb.block_count, decoded.block_count); - try std.testing.expectEqual(sb.journal_head, decoded.journal_head); -} - -test "block storage create and open" { - const allocator = std.testing.allocator; - const path = "test_blocks.lgh"; - defer std.fs.cwd().deleteFile(path) catch {}; - - // Create new storage - { - const storage = try BlockStorage.open(allocator, path); - defer storage.deinit(); - - try std.testing.expectEqual(@as(u64, 1), storage.superblock.block_count); - } - - // Reopen existing - { - const storage = try BlockStorage.open(allocator, path); - defer storage.deinit(); - - try std.testing.expectEqual(@as(u64, 1), storage.superblock.block_count); - } -} - -test "block allocation and write" { - const allocator = std.testing.allocator; - const path = "test_alloc.lgh"; - defer std.fs.cwd().deleteFile(path) catch {}; - - const storage = try BlockStorage.open(allocator, path); - defer storage.deinit(); - - // Allocate and write a document block - const block_id = try storage.allocateBlock(.document); - try std.testing.expectEqual(@as(u64, 1), block_id); - - var block = try storage.readBlock(block_id); - try block.setPayload("test document"); - try storage.writeBlock(block_id, &block); - - // Read it back - const read_block = try storage.readBlock(block_id); - try std.testing.expectEqualSlices(u8, "test document", read_block.getPayload()); -} - -test "journal append" { - const allocator = std.testing.allocator; - const path = "test_journal.lgh"; - defer std.fs.cwd().deleteFile(path) catch {}; - - const storage = try BlockStorage.open(allocator, path); - defer storage.deinit(); - - // Append journal entries - const j1 = try storage.appendJournal("entry 1"); - const j2 = try storage.appendJournal("entry 2"); - - // Verify linkage - const block2 = try storage.readBlock(j2); - try std.testing.expectEqual(j1, block2.header.prev_block_id); -} diff --git a/lithoglyph/core-zig/src/bridge.zig b/lithoglyph/core-zig/src/bridge.zig deleted file mode 100644 index 4cfc8f7f..00000000 --- a/lithoglyph/core-zig/src/bridge.zig +++ /dev/null @@ -1,1145 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith.Bridge - C ABI Layer -// -// Provides stable C-compatible API for runtimes to interact with Lithoglyph. -// All blob arguments and return values use CBOR encoding. -// -// Part of Lithoglyph: Stone-carved data for the ages. -// Lg* = Lithoglyph types (abbreviated for C compatibility) - -const std = @import("std"); -const blocks = @import("blocks.zig"); - -// Simplified types for C ABI (no external dependencies) -pub const LgBlob = extern struct { - ptr: ?[*]const u8, - len: usize, - - pub fn empty() LgBlob { - return .{ .ptr = null, .len = 0 }; - } - - pub fn fromSlice(slice: []const u8) LgBlob { - return .{ .ptr = slice.ptr, .len = slice.len }; - } - - pub fn toSlice(self: LgBlob) ?[]const u8 { - if (self.ptr) |ptr| { - return ptr[0..self.len]; - } - return null; - } -}; - -pub const LgStatus = enum(c_int) { - ok = 0, - err_internal = 1, - err_not_found = 2, - err_invalid_argument = 3, - err_out_of_memory = 4, - err_not_implemented = 5, - err_txn_not_active = 6, - err_txn_already_committed = 7, -}; - -pub const LgResult = extern struct { - data: LgBlob, - provenance: LgBlob, - status: LgStatus, - error_blob: LgBlob, - - pub fn ok(data_blob: LgBlob) LgResult { - return .{ - .data = data_blob, - .provenance = LgBlob.empty(), - .status = .ok, - .error_blob = LgBlob.empty(), - }; - } - - pub fn okWithProvenance(data_blob: LgBlob, prov_blob: LgBlob) LgResult { - return .{ - .data = data_blob, - .provenance = prov_blob, - .status = .ok, - .error_blob = LgBlob.empty(), - }; - } - - pub fn err(status: LgStatus, error_blob: LgBlob) LgResult { - return .{ - .data = LgBlob.empty(), - .provenance = LgBlob.empty(), - .status = status, - .error_blob = error_blob, - }; - } -}; - -pub const LgTxnMode = enum(c_int) { - read_only = 0, - read_write = 1, -}; - -pub const LgRenderOpts = extern struct { - format: c_int, - include_metadata: bool, -}; - -// ============================================================ -// Opaque Handles -// ============================================================ - -pub const LgDb = opaque {}; -pub const LgTxn = opaque {}; - -// Internal state structures -const DbState = struct { - allocator: std.mem.Allocator, - storage: *blocks.BlockStorage, -}; - -/// A pending write operation buffered within a transaction -const PendingWrite = struct { - block_id: u64, - data: []u8, // owned copy of payload - journal_msg: []u8, // owned journal entry text - is_new: bool, // true=insert, false=update -}; - -const TxnState = struct { - db: *DbState, - mode: LgTxnMode, - is_active: bool, - sequence: u64, - pending_writes: std.ArrayList(PendingWrite), - pending_deletes: std.ArrayList(u64), // block IDs to delete - - fn deinitPending(self: *TxnState) void { - for (self.pending_writes.items) |pw| { - global_allocator.free(pw.data); - global_allocator.free(pw.journal_msg); - } - self.pending_writes.deinit(global_allocator); - self.pending_deletes.deinit(global_allocator); - } -}; - -// Global allocator for C ABI (can't pass allocator through C) -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -const global_allocator = gpa.allocator(); - -// Active handles registry -var db_registry = std.AutoHashMap(*DbState, void).init(global_allocator); -var txn_registry = std.AutoHashMap(*TxnState, void).init(global_allocator); - -// ============================================================ -// Error Blob Creation -// ============================================================ - -fn createErrorBlob(status: LgStatus, message: []const u8) LgBlob { - // Format error as simple JSON - var buf: [512]u8 = undefined; - const err_str = std.fmt.bufPrint(&buf, - \\{{"status":{d},"error":"{s}"}} - , .{ @intFromEnum(status), message }) catch return LgBlob.empty(); - - const err_data = global_allocator.dupe(u8, err_str) catch return LgBlob.empty(); - return LgBlob.fromSlice(err_data); -} - -// ============================================================ -// Database Lifecycle - C ABI Exports -// ============================================================ - -/// Open a Lith database -/// -/// @param path_ptr Path to database file -/// @param path_len Length of path -/// @param opts_ptr CBOR-encoded options (nullable) -/// @param opts_len Length of options -/// @param out_db Output parameter for database handle -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_db_open( - path_ptr: [*]const u8, - path_len: usize, - opts_ptr: ?[*]const u8, - opts_len: usize, - out_db: *?*LgDb, - out_err: *LgBlob, -) LgStatus { - _ = opts_ptr; - _ = opts_len; - - const path = path_ptr[0..path_len]; - - // Open or create block storage - const storage = blocks.BlockStorage.open(global_allocator, path) catch |err| { - const msg = switch (err) { - error.OutOfMemory => "Out of memory", - error.FileNotFound => "Database not found", - error.InvalidDatabase => "Invalid database format", - else => "Failed to open database", - }; - out_err.* = createErrorBlob(.err_internal, msg); - return .err_internal; - }; - - // Create database state - const db = global_allocator.create(DbState) catch { - storage.deinit(); - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate database state"); - return .err_out_of_memory; - }; - - db.* = .{ - .allocator = global_allocator, - .storage = storage, - }; - - // Register handle - db_registry.put(db, {}) catch { - storage.deinit(); - global_allocator.destroy(db); - out_err.* = createErrorBlob(.err_internal, "Failed to register database handle"); - return .err_internal; - }; - - // SAFETY: db is a freshly-allocated *DbState from global_allocator.create() above, - // registered in db_registry. Cast to opaque *LgDb is safe because callers only - // pass it back through C ABI functions that cast it back to *DbState after - // validating the handle exists in db_registry. - out_db.* = @ptrCast(db); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Close a Lith database -/// -/// @param db Database handle -/// @return Status code -pub export fn lith_db_close(db: ?*LgDb) LgStatus { - // SAFETY: db was originally a *DbState allocated by global_allocator.create() - // in lith_db_open, then cast to *LgDb. The orelse guards null. Alignment is - // guaranteed because DbState was heap-allocated with proper alignment by the GPA. - // The subsequent db_registry.contains() check validates the pointer is still live. - const state: *DbState = @ptrCast(@alignCast(db orelse return .err_invalid_argument)); - - if (!db_registry.contains(state)) { - return .err_invalid_argument; - } - - // Clean up any active transactions - var txn_iter = txn_registry.keyIterator(); - while (txn_iter.next()) |txn| { - if (txn.*.db == state) { - _ = txn_registry.remove(txn.*); - global_allocator.destroy(txn.*); - } - } - - // Close block storage - state.storage.deinit(); - - // Clean up database state - _ = db_registry.remove(state); - global_allocator.destroy(state); - - return .ok; -} - -// ============================================================ -// Transaction Management - C ABI Exports -// ============================================================ - -/// Begin a new transaction -/// -/// @param db Database handle -/// @param mode Transaction mode (read-only or read-write) -/// @param out_txn Output parameter for transaction handle -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_txn_begin( - db: ?*LgDb, - mode: LgTxnMode, - out_txn: *?*LgTxn, - out_err: *LgBlob, -) LgStatus { - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create() which respects @alignOf(DbState). The - // db_registry.contains() check below validates the pointer is still registered. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Create transaction state - const txn = global_allocator.create(TxnState) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate transaction"); - return .err_out_of_memory; - }; - - txn.* = .{ - .db = state, - .mode = mode, - .is_active = true, - .sequence = state.storage.superblock.journal_head + 1, - .pending_writes = .{}, - .pending_deletes = .{}, - }; - - txn_registry.put(txn, {}) catch { - global_allocator.destroy(txn); - out_err.* = createErrorBlob(.err_internal, "Failed to register transaction"); - return .err_internal; - }; - - // SAFETY: txn is a freshly-allocated *TxnState from global_allocator.create() - // above, registered in txn_registry. Cast to opaque *LgTxn is safe because - // callers only pass it back through C ABI functions that cast it back to - // *TxnState after validating the handle exists in txn_registry. - out_txn.* = @ptrCast(txn); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Commit a transaction -/// -/// @param txn Transaction handle -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_txn_commit(txn: ?*LgTxn, out_err: *LgBlob) LgStatus { - // SAFETY: txn was originally a *TxnState from lith_txn_begin, cast to opaque - // *LgTxn. The orelse guards null. Alignment is safe because TxnState was - // heap-allocated by global_allocator.create(). The txn_registry.contains() - // check below validates the pointer is still a live, registered handle. - const state: *TxnState = @ptrCast(@alignCast(txn orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid transaction handle"); - return .err_invalid_argument; - })); - - if (!txn_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Transaction handle not registered"); - return .err_invalid_argument; - } - - if (!state.is_active) { - out_err.* = createErrorBlob(.err_txn_already_committed, "Transaction already committed"); - return .err_txn_already_committed; - } - - // Atomic commit with WAL ordering: - // Phase 1: Write all journal entries (WAL — durable before data) - for (state.pending_writes.items) |pw| { - _ = state.db.storage.appendJournal(pw.journal_msg) catch { - out_err.* = createErrorBlob(.err_internal, "Journal write failed during commit"); - return .err_internal; - }; - } - for (state.pending_deletes.items) |block_id| { - var del_buf: [80]u8 = undefined; - const del_msg = std.fmt.bufPrint(&del_buf, "DELETE block_id={d}", .{block_id}) catch continue; - _ = state.db.storage.appendJournal(del_msg) catch { - out_err.* = createErrorBlob(.err_internal, "Journal write failed during commit"); - return .err_internal; - }; - } - - // Phase 2: Sync journal to disk (WAL guarantee) - state.db.storage.file.sync() catch {}; - - // Phase 3: Write all data blocks - for (state.pending_writes.items) |pw| { - var block = blocks.Block.init(.document, pw.block_id, state.sequence); - block.setPayload(pw.data) catch continue; - state.db.storage.writeBlock(pw.block_id, &block) catch { - out_err.* = createErrorBlob(.err_internal, "Block write failed during commit"); - return .err_internal; - }; - } - - // Phase 4: Process deletions - for (state.pending_deletes.items) |block_id| { - state.db.storage.freeBlock(block_id) catch continue; - } - - // Phase 5: Flush superblock (reflects all allocations) - state.db.storage.flushSuperblock() catch {}; - - // Phase 6: Final sync (all data durable) - state.db.storage.file.sync() catch {}; - - // Clean up transaction - state.deinitPending(); - state.is_active = false; - _ = txn_registry.remove(state); - global_allocator.destroy(state); - - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Abort a transaction -/// -/// @param txn Transaction handle -/// @return Status code -pub export fn lith_txn_abort(txn: ?*LgTxn) LgStatus { - // SAFETY: txn was originally a *TxnState from lith_txn_begin, cast to opaque - // *LgTxn. The orelse guards null. Alignment is safe because TxnState was - // heap-allocated by global_allocator.create(). The txn_registry.contains() - // check below validates the pointer is still a live, registered handle. - const state: *TxnState = @ptrCast(@alignCast(txn orelse return .err_invalid_argument)); - - if (!txn_registry.contains(state)) { - return .err_invalid_argument; - } - - // Discard all buffered operations (nothing was written to disk) - state.deinitPending(); - state.is_active = false; - - // Clean up transaction - _ = txn_registry.remove(state); - global_allocator.destroy(state); - - return .ok; -} - -// ============================================================ -// Operations - C ABI Exports -// ============================================================ - -/// Apply an operation within a transaction -/// -/// @param txn Transaction handle -/// @param op_ptr Raw data to store -/// @param op_len Length of data -/// @return Result containing block ID and status -/// Apply an operation within a transaction (buffered — not written until commit) -pub export fn lith_apply( - txn: ?*LgTxn, - op_ptr: [*]const u8, - op_len: usize, -) LgResult { - // SAFETY: txn was originally a *TxnState from lith_txn_begin, cast to opaque - // *LgTxn. The orelse guards null. Alignment is safe because TxnState was - // heap-allocated by global_allocator.create(). The txn_registry.contains() - // check below validates the pointer is still a live, registered handle. - const state: *TxnState = @ptrCast(@alignCast(txn orelse { - return LgResult.err(.err_invalid_argument, createErrorBlob(.err_invalid_argument, "Invalid transaction")); - })); - - if (!txn_registry.contains(state)) { - return LgResult.err(.err_invalid_argument, createErrorBlob(.err_invalid_argument, "Transaction not registered")); - } - - if (!state.is_active) { - return LgResult.err(.err_txn_not_active, createErrorBlob(.err_txn_not_active, "Transaction not active")); - } - - if (state.mode != .read_write) { - return LgResult.err(.err_invalid_argument, createErrorBlob(.err_invalid_argument, "Read-only transaction")); - } - - const op_data = op_ptr[0..op_len]; - - // Validate payload fits in a block - if (op_len > blocks.PAYLOAD_SIZE) { - return LgResult.err(.err_invalid_argument, createErrorBlob(.err_invalid_argument, "Payload too large for single block")); - } - - // Reserve a block ID (memory only — no disk write yet) - const block_id = state.db.storage.reserveBlockId(); - - // Copy payload data (owned by transaction until commit/abort) - const data_copy = global_allocator.dupe(u8, op_data) catch { - return LgResult.err(.err_out_of_memory, LgBlob.empty()); - }; - - // Format journal entry - var journal_buf: [100]u8 = undefined; - const journal_str = std.fmt.bufPrint(&journal_buf, "INSERT block_id={d} size={d}", .{ block_id, op_len }) catch { - global_allocator.free(data_copy); - return LgResult.err(.err_internal, createErrorBlob(.err_internal, "Failed to format journal")); - }; - - const journal_copy = global_allocator.dupe(u8, journal_str) catch { - global_allocator.free(data_copy); - return LgResult.err(.err_out_of_memory, LgBlob.empty()); - }; - - // Buffer the write (deferred until commit) - state.pending_writes.append(global_allocator, .{ - .block_id = block_id, - .data = data_copy, - .journal_msg = journal_copy, - .is_new = true, - }) catch { - global_allocator.free(data_copy); - global_allocator.free(journal_copy); - return LgResult.err(.err_out_of_memory, LgBlob.empty()); - }; - - // Return block ID as result (operation is pending, not yet durable) - var result_buf: [80]u8 = undefined; - const result_str = std.fmt.bufPrint(&result_buf, - \\{{"block_id":{d},"status":"pending"}} - , .{block_id}) catch { - return LgResult.err(.err_internal, createErrorBlob(.err_internal, "Failed to format result")); - }; - - const result_data = global_allocator.dupe(u8, result_str) catch { - return LgResult.err(.err_out_of_memory, LgBlob.empty()); - }; - - return LgResult.ok(LgBlob.fromSlice(result_data)); -} - -/// Update an existing block within a transaction (buffered) -pub export fn lith_update_block( - txn: ?*LgTxn, - block_id: u64, - data_ptr: [*]const u8, - data_len: usize, - out_err: *LgBlob, -) LgStatus { - // SAFETY: txn was originally a *TxnState from lith_txn_begin, cast to opaque - // *LgTxn. The orelse guards null. Alignment is safe because TxnState was - // heap-allocated by global_allocator.create(). State validity is checked - // immediately after via is_active and mode fields. - const state: *TxnState = @ptrCast(@alignCast(txn orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid transaction"); - return .err_invalid_argument; - })); - - if (!state.is_active or state.mode != .read_write) { - out_err.* = createErrorBlob(.err_txn_not_active, "Transaction not active or read-only"); - return .err_txn_not_active; - } - - if (data_len > blocks.PAYLOAD_SIZE) { - out_err.* = createErrorBlob(.err_invalid_argument, "Payload too large"); - return .err_invalid_argument; - } - - const data_copy = global_allocator.dupe(u8, data_ptr[0..data_len]) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - var journal_buf: [100]u8 = undefined; - const journal_str = std.fmt.bufPrint(&journal_buf, "UPDATE block_id={d} size={d}", .{ block_id, data_len }) catch { - global_allocator.free(data_copy); - out_err.* = createErrorBlob(.err_internal, "Failed to format journal"); - return .err_internal; - }; - - const journal_copy = global_allocator.dupe(u8, journal_str) catch { - global_allocator.free(data_copy); - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - state.pending_writes.append(global_allocator, .{ - .block_id = block_id, - .data = data_copy, - .journal_msg = journal_copy, - .is_new = false, - }) catch { - global_allocator.free(data_copy); - global_allocator.free(journal_copy); - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Delete a block within a transaction (buffered) -pub export fn lith_delete_block( - txn: ?*LgTxn, - block_id: u64, - out_err: *LgBlob, -) LgStatus { - // SAFETY: txn was originally a *TxnState from lith_txn_begin, cast to opaque - // *LgTxn. The orelse guards null. Alignment is safe because TxnState was - // heap-allocated by global_allocator.create(). State validity is checked - // immediately after via is_active and mode fields. - const state: *TxnState = @ptrCast(@alignCast(txn orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid transaction"); - return .err_invalid_argument; - })); - - if (!state.is_active or state.mode != .read_write) { - out_err.* = createErrorBlob(.err_txn_not_active, "Transaction not active or read-only"); - return .err_txn_not_active; - } - - state.pending_deletes.append(global_allocator, block_id) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Read all blocks of a given type (full scan for PoC) -/// Returns JSON array of objects with block_id and data fields. -pub export fn lith_read_blocks( - db: ?*LgDb, - block_type: u16, - out_data: *LgBlob, - out_err: *LgBlob, -) LgStatus { - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create(). The db_registry.contains() check below validates - // the pointer is still a live, registered handle. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Build JSON array by scanning all blocks - var result: std.ArrayList(u8) = .{}; - defer result.deinit(global_allocator); - - result.appendSlice(global_allocator, "[") catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - var first = true; - var block_id: u64 = 1; - while (block_id < state.storage.superblock.block_count) : (block_id += 1) { - const block = state.storage.readBlock(block_id) catch continue; - - // Filter by type and skip deleted blocks - if (block.header.block_type != block_type) continue; - if (block.header.flags & 0x08 != 0) continue; // FLAG_DELETED - - if (!first) { - result.appendSlice(global_allocator, ",") catch continue; - } - first = false; - - // Format as JSON object with block_id and raw payload - const payload = block.getPayload(); - - // Start JSON object - var header_buf: [80]u8 = undefined; - const header_str = std.fmt.bufPrint(&header_buf, - \\{{"block_id":{d},"size":{d},"data": - , .{ block.header.block_id, block.header.payload_len }) catch continue; - - result.appendSlice(global_allocator, header_str) catch continue; - - // Include payload as JSON-escaped string - result.appendSlice(global_allocator, "\"") catch continue; - for (payload) |byte| { - switch (byte) { - '"' => result.appendSlice(global_allocator, "\\\"") catch continue, - '\\' => result.appendSlice(global_allocator, "\\\\") catch continue, - '\n' => result.appendSlice(global_allocator, "\\n") catch continue, - '\r' => result.appendSlice(global_allocator, "\\r") catch continue, - '\t' => result.appendSlice(global_allocator, "\\t") catch continue, - else => { - if (byte >= 0x20 and byte < 0x7F) { - result.append(global_allocator, byte) catch continue; - } else { - var hex_buf: [6]u8 = undefined; - const hex_str = std.fmt.bufPrint(&hex_buf, "\\u{x:0>4}", .{byte}) catch continue; - result.appendSlice(global_allocator, hex_str) catch continue; - } - }, - } - } - result.appendSlice(global_allocator, "\"}") catch continue; - } - - result.appendSlice(global_allocator, "]") catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - const result_data = global_allocator.dupe(u8, result.items) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Out of memory"); - return .err_out_of_memory; - }; - - out_data.* = LgBlob.fromSlice(result_data); - out_err.* = LgBlob.empty(); - return .ok; -} - -// ============================================================ -// Introspection - C ABI Exports -// ============================================================ - -/// Render a block as canonical text -/// -/// @param db Database handle -/// @param block_id Block ID to render -/// @param opts Render options -/// @param out_text Output parameter for text blob -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_render_block( - db: ?*LgDb, - block_id: u64, - opts: LgRenderOpts, - out_text: *LgBlob, - out_err: *LgBlob, -) LgStatus { - _ = opts; - - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create(). The db_registry.contains() check below validates - // the pointer is still a live, registered handle. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Read block from storage - const block = state.storage.readBlock(block_id) catch |err| { - const msg = switch (err) { - error.InvalidBlock => "Block not found or invalid", - error.ChecksumMismatch => "Block checksum mismatch", - else => "Failed to read block", - }; - out_err.* = createErrorBlob(.err_internal, msg); - return .err_internal; - }; - - // Format block as JSON (show payload size only, not content) - _ = block.getPayload(); // Validate payload exists - var buf: [8192]u8 = undefined; - const text = std.fmt.bufPrint(&buf, - \\{{"block_id":{d},"type":"{s}","sequence":{d},"size":{d},"payload":"[{d} bytes]"}} - , .{ - block.header.block_id, - @tagName(@as(blocks.BlockType, @enumFromInt(block.header.block_type))), - block.header.sequence, - block.header.payload_len, - block.header.payload_len, - }) catch { - out_err.* = createErrorBlob(.err_internal, "Failed to format block"); - return .err_internal; - }; - - const text_data = global_allocator.dupe(u8, text) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate result"); - return .err_out_of_memory; - }; - - out_text.* = LgBlob.fromSlice(text_data); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Render journal entries since a sequence number -/// -/// @param db Database handle -/// @param since Sequence number to start from -/// @param opts Render options -/// @param out_text Output parameter for text blob -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_render_journal( - db: ?*LgDb, - since: u64, - opts: LgRenderOpts, - out_text: *LgBlob, - out_err: *LgBlob, -) LgStatus { - _ = opts; - - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create(). The db_registry.contains() check below validates - // the pointer is still a live, registered handle. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Format journal info as JSON - var buf: [512]u8 = undefined; - const text = std.fmt.bufPrint(&buf, - \\{{"since":{d},"head":{d},"tail":{d},"entries":[]}} - , .{ - since, - state.storage.superblock.journal_head, - state.storage.superblock.journal_tail, - }) catch { - out_err.* = createErrorBlob(.err_internal, "Failed to format journal"); - return .err_internal; - }; - - const text_data = global_allocator.dupe(u8, text) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate result"); - return .err_out_of_memory; - }; - - out_text.* = LgBlob.fromSlice(text_data); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Get database schema information -/// -/// @param db Database handle -/// @param out_schema Output parameter for schema blob -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_introspect_schema( - db: ?*LgDb, - out_schema: *LgBlob, - out_err: *LgBlob, -) LgStatus { - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create(). The db_registry.contains() check below validates - // the pointer is still a live, registered handle. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Format schema as JSON - var buf: [512]u8 = undefined; - const schema_str = std.fmt.bufPrint(&buf, - \\{{"version":{d},"block_count":{d},"collections":[]}} - , .{ - state.storage.superblock.version, - state.storage.superblock.block_count, - }) catch { - out_err.* = createErrorBlob(.err_internal, "Failed to format schema"); - return .err_internal; - }; - - const schema_data = global_allocator.dupe(u8, schema_str) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate result"); - return .err_out_of_memory; - }; - - out_schema.* = LgBlob.fromSlice(schema_data); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Get constraint information -/// -/// @param db Database handle -/// @param out_constraints Output parameter for constraints blob -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_introspect_constraints( - db: ?*LgDb, - out_constraints: *LgBlob, - out_err: *LgBlob, -) LgStatus { - // SAFETY: db was originally a *DbState from lith_db_open, cast to opaque *LgDb. - // The orelse guards null. Alignment is safe because DbState was heap-allocated - // by global_allocator.create(). The db_registry.contains() check below validates - // the pointer is still a live, registered handle. - const state: *DbState = @ptrCast(@alignCast(db orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid database handle"); - return .err_invalid_argument; - })); - - if (!db_registry.contains(state)) { - out_err.* = createErrorBlob(.err_invalid_argument, "Database handle not registered"); - return .err_invalid_argument; - } - - // Generate constraint introspection (placeholder - no constraints yet) - const constraint_json = "{\"constraints\":[],\"functional_dependencies\":[]}"; - const constraint_data = global_allocator.dupe(u8, constraint_json) catch { - out_err.* = createErrorBlob(.err_out_of_memory, "Failed to allocate result"); - return .err_out_of_memory; - }; - - out_constraints.* = LgBlob.fromSlice(constraint_data); - out_err.* = LgBlob.empty(); - return .ok; -} - -// ============================================================ -// Proof Verification (per D-NORM-004) -// ============================================================ - -/// Proof verifier callback type -pub const LgProofVerifier = *const fn ( - proof_ptr: [*]const u8, - proof_len: usize, - context_ptr: ?*anyopaque, -) callconv(.c) LgStatus; - -/// Proof verifier registration entry -const VerifierEntry = struct { - verifier_type: []const u8, - callback: LgProofVerifier, - context: ?*anyopaque, -}; - -// Registry of proof verifiers -var verifier_registry = std.StringHashMap(VerifierEntry).init(global_allocator); - -/// Register a proof verifier for a specific proof type -/// -/// @param type_ptr Proof type identifier (e.g., "normalization", "fd-holds") -/// @param type_len Length of type identifier -/// @param callback Verification function -/// @param context Optional context passed to callback -/// @return Status code -pub export fn lith_proof_register_verifier( - type_ptr: [*]const u8, - type_len: usize, - callback: LgProofVerifier, - context: ?*anyopaque, -) LgStatus { - const verifier_type = type_ptr[0..type_len]; - - const type_copy = global_allocator.dupe(u8, verifier_type) catch { - return .err_out_of_memory; - }; - - const entry = VerifierEntry{ - .verifier_type = type_copy, - .callback = callback, - .context = context, - }; - - verifier_registry.put(type_copy, entry) catch { - global_allocator.free(type_copy); - return .err_internal; - }; - - return .ok; -} - -/// Unregister a proof verifier -/// -/// @param type_ptr Proof type identifier -/// @param type_len Length of type identifier -/// @return Status code -pub export fn lith_proof_unregister_verifier( - type_ptr: [*]const u8, - type_len: usize, -) LgStatus { - const verifier_type = type_ptr[0..type_len]; - - if (verifier_registry.fetchRemove(verifier_type)) |entry| { - global_allocator.free(@constCast(entry.value.verifier_type)); - return .ok; - } - - return .err_not_found; -} - -/// Verify a proof using registered verifiers -/// -/// @param proof_ptr CBOR-encoded proof blob -/// @param proof_len Length of proof -/// @param out_valid Output: true if proof is valid -/// @param out_err Output parameter for error blob -/// @return Status code -pub export fn lith_proof_verify( - proof_ptr: [*]const u8, - proof_len: usize, - out_valid: *bool, - out_err: *LgBlob, -) LgStatus { - const proof_data = proof_ptr[0..proof_len]; - - // Parse JSON proof to extract type and data - // Expected format: {"type":"proof_type","data":"base64_data"} - const parsed = std.json.parseFromSlice( - std.json.Value, - global_allocator, - proof_data, - .{}, - ) catch { - out_err.* = createErrorBlob(.err_invalid_argument, "Invalid JSON proof format"); - return .err_invalid_argument; - }; - defer parsed.deinit(); - - const root = parsed.value; - if (root != .object) { - out_err.* = createErrorBlob(.err_invalid_argument, "Proof must be JSON object"); - return .err_invalid_argument; - } - - const type_value = root.object.get("type") orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Proof missing 'type' field"); - return .err_invalid_argument; - }; - - if (type_value != .string) { - out_err.* = createErrorBlob(.err_invalid_argument, "Proof 'type' must be string"); - return .err_invalid_argument; - } - - const ptype = type_value.string; - - const entry = verifier_registry.get(ptype) orelse { - out_err.* = createErrorBlob(.err_not_found, "No verifier registered for proof type"); - return .err_not_found; - }; - - // Extract proof data (as string for now) - const data_value = root.object.get("data") orelse { - out_err.* = createErrorBlob(.err_invalid_argument, "Proof missing 'data' field"); - return .err_invalid_argument; - }; - - if (data_value != .string) { - out_err.* = createErrorBlob(.err_invalid_argument, "Proof 'data' must be string"); - return .err_invalid_argument; - } - - const verify_data = data_value.string; - const status = entry.callback(verify_data.ptr, verify_data.len, entry.context); - - out_valid.* = (status == .ok); - out_err.* = LgBlob.empty(); - return .ok; -} - -/// Built-in verifier for FD-holds proofs (always accepts for PoC) -fn builtin_fd_verifier( - _: [*]const u8, - _: usize, - _: ?*anyopaque, -) callconv(.c) LgStatus { - // In production, this would actually verify the proof - // For PoC, we accept all well-formed proofs - return .ok; -} - -/// Built-in verifier for normalization proofs -fn builtin_normalization_verifier( - _: [*]const u8, - _: usize, - _: ?*anyopaque, -) callconv(.c) LgStatus { - // In production, this would verify losslessness and dependency preservation - // For PoC, we accept all well-formed proofs - return .ok; -} - -/// Initialize built-in proof verifiers -pub export fn lith_proof_init_builtins() LgStatus { - // Register FD-holds verifier - const fd_type = "fd-holds"; - var status = lith_proof_register_verifier(fd_type.ptr, fd_type.len, builtin_fd_verifier, null); - if (status != .ok) return status; - - // Register normalization verifier - const norm_type = "normalization"; - status = lith_proof_register_verifier(norm_type.ptr, norm_type.len, builtin_normalization_verifier, null); - if (status != .ok) return status; - - // Register denormalization verifier (same logic) - const denorm_type = "denormalization"; - status = lith_proof_register_verifier(denorm_type.ptr, denorm_type.len, builtin_normalization_verifier, null); - - return status; -} - -// ============================================================ -// Utility Functions - C ABI Exports -// ============================================================ - -/// Free a blob allocated by the bridge -/// -/// @param blob Blob to free -pub export fn lith_blob_free(blob: *LgBlob) void { - if (blob.toSlice()) |slice| { - global_allocator.free(@constCast(slice)); - } - blob.* = LgBlob.empty(); -} - -/// Get Lith version -/// -/// @return Version as encoded integer (major * 10000 + minor * 100 + patch) -pub export fn lith_version() u32 { - return 0 * 10000 + 1 * 100 + 0; // 0.1.0 -} - -// ============================================================ -// Tests -// ============================================================ - -test "database lifecycle" { - var db: ?*LgDb = null; - var err_blob: LgBlob = undefined; - - const path = "test.lgh"; - const status = lith_db_open(path.ptr, path.len, null, 0, &db, &err_blob); - - try std.testing.expectEqual(LgStatus.ok, status); - try std.testing.expect(db != null); - - const close_status = lith_db_close(db); - try std.testing.expectEqual(LgStatus.ok, close_status); -} - -test "transaction lifecycle" { - var db: ?*LgDb = null; - var err_blob: LgBlob = undefined; - - const path = "test_txn.lgh"; - _ = lith_db_open(path.ptr, path.len, null, 0, &db, &err_blob); - defer _ = lith_db_close(db); - - var txn: ?*LgTxn = null; - var txn_err: LgBlob = undefined; - - const begin_status = lith_txn_begin(db, .read_write, &txn, &txn_err); - try std.testing.expectEqual(LgStatus.ok, begin_status); - try std.testing.expect(txn != null); - - var commit_err: LgBlob = undefined; - const commit_status = lith_txn_commit(txn, &commit_err); - try std.testing.expectEqual(LgStatus.ok, commit_status); -} - -test "version" { - const version = lith_version(); - try std.testing.expectEqual(@as(u32, 100), version); // 0.1.0 -} diff --git a/lithoglyph/core-zig/src/cbor.zig b/lithoglyph/core-zig/src/cbor.zig deleted file mode 100644 index aa73864b..00000000 --- a/lithoglyph/core-zig/src/cbor.zig +++ /dev/null @@ -1,519 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Form.Bridge - CBOR Encoding/Decoding -// -// Minimal CBOR implementation following RFC 8949. -// Supports deterministic encoding per Section 4.2. -// -// Part of Lithoglyph: Stone-carved data for the ages. - -const std = @import("std"); -const types = @import("types.zig"); - -// ============================================================ -// CBOR Major Types -// ============================================================ - -pub const MajorType = enum(u3) { - unsigned = 0, - negative = 1, - bytes = 2, - text = 3, - array = 4, - map = 5, - tag = 6, - simple = 7, -}; - -// ============================================================ -// CBOR Encoder -// ============================================================ - -pub const Encoder = struct { - buffer: std.ArrayList(u8), - - pub fn init(allocator: std.mem.Allocator) Encoder { - return .{ - .buffer = std.ArrayList(u8).init(allocator), - }; - } - - pub fn deinit(self: *Encoder) void { - self.buffer.deinit(); - } - - pub fn finish(self: *Encoder) []const u8 { - return self.buffer.items; - } - - pub fn reset(self: *Encoder) void { - self.buffer.clearRetainingCapacity(); - } - - // Write major type with argument - fn writeTypeArg(self: *Encoder, major: MajorType, arg: u64) !void { - const base: u8 = @as(u8, @intFromEnum(major)) << 5; - - if (arg < 24) { - try self.buffer.append(base | @as(u8, @truncate(arg))); - } else if (arg <= 0xFF) { - try self.buffer.append(base | 24); - try self.buffer.append(@truncate(arg)); - } else if (arg <= 0xFFFF) { - try self.buffer.append(base | 25); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u16, @truncate(arg)))); - } else if (arg <= 0xFFFFFFFF) { - try self.buffer.append(base | 26); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u32, @truncate(arg)))); - } else { - try self.buffer.append(base | 27); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u64, arg))); - } - } - - // Encode unsigned integer - pub fn encodeUint(self: *Encoder, value: u64) !void { - try self.writeTypeArg(.unsigned, value); - } - - // Encode negative integer - pub fn encodeNint(self: *Encoder, value: i64) !void { - const n: u64 = @bitCast(-1 - value); - try self.writeTypeArg(.negative, n); - } - - // Encode integer (signed or unsigned) - pub fn encodeInt(self: *Encoder, value: i64) !void { - if (value >= 0) { - try self.encodeUint(@bitCast(value)); - } else { - try self.encodeNint(value); - } - } - - // Encode byte string - pub fn encodeBytes(self: *Encoder, data: []const u8) !void { - try self.writeTypeArg(.bytes, data.len); - try self.buffer.appendSlice(data); - } - - // Encode text string - pub fn encodeText(self: *Encoder, text: []const u8) !void { - try self.writeTypeArg(.text, text.len); - try self.buffer.appendSlice(text); - } - - // Begin array (definite length) - pub fn beginArray(self: *Encoder, len: usize) !void { - try self.writeTypeArg(.array, len); - } - - // Begin map (definite length) - pub fn beginMap(self: *Encoder, len: usize) !void { - try self.writeTypeArg(.map, len); - } - - // Encode tag - pub fn encodeTag(self: *Encoder, tag: u64) !void { - try self.writeTypeArg(.tag, tag); - } - - // Encode Lith-specific tag - pub fn encodeLithTag(self: *Encoder, tag: types.CborTag) !void { - try self.encodeTag(@intFromEnum(tag)); - } - - // Encode null - pub fn encodeNull(self: *Encoder) !void { - try self.buffer.append(0xF6); - } - - // Encode boolean - pub fn encodeBool(self: *Encoder, value: bool) !void { - try self.buffer.append(if (value) 0xF5 else 0xF4); - } - - // Encode float (smallest representation per RFC 8949 §4.2) - pub fn encodeFloat(self: *Encoder, value: f64) !void { - // Check if it fits in half precision - const half: f16 = @floatCast(value); - if (@as(f64, @floatCast(half)) == value) { - try self.buffer.append(0xF9); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u16, @bitCast(half)))); - return; - } - - // Check if it fits in single precision - const single: f32 = @floatCast(value); - if (@as(f64, @floatCast(single)) == value) { - try self.buffer.append(0xFA); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u32, @bitCast(single)))); - return; - } - - // Use double precision - try self.buffer.append(0xFB); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u64, @bitCast(value)))); - } - - // Encode a simple document (map of string -> any) - pub fn encodeDocument(self: *Encoder, fields: anytype) !void { - const info = @typeInfo(@TypeOf(fields)); - const struct_info = info.@"struct"; - - try self.beginMap(struct_info.fields.len); - - inline for (struct_info.fields) |field| { - try self.encodeText(field.name); - const value = @field(fields, field.name); - try self.encodeValue(value); - } - } - - // Encode any value (comptime type dispatch) - pub fn encodeValue(self: *Encoder, value: anytype) !void { - const T = @TypeOf(value); - - if (T == bool) { - try self.encodeBool(value); - } else if (@typeInfo(T) == .int) { - try self.encodeInt(@intCast(value)); - } else if (@typeInfo(T) == .float) { - try self.encodeFloat(@floatCast(value)); - } else if (T == []const u8) { - try self.encodeText(value); - } else if (@typeInfo(T) == .pointer) { - if (@typeInfo(T).pointer.size == .Slice) { - if (@typeInfo(T).pointer.child == u8) { - try self.encodeText(value); - } else { - try self.beginArray(value.len); - for (value) |item| { - try self.encodeValue(item); - } - } - } - } else if (@typeInfo(T) == .optional) { - if (value) |v| { - try self.encodeValue(v); - } else { - try self.encodeNull(); - } - } else if (@typeInfo(T) == .@"struct") { - try self.encodeDocument(value); - } else { - @compileError("unsupported type for CBOR encoding: " ++ @typeName(T)); - } - } -}; - -// ============================================================ -// CBOR Decoder -// ============================================================ - -pub const DecodeError = error{ - UnexpectedEof, - InvalidType, - InvalidValue, - OutOfMemory, -}; - -pub const Decoder = struct { - data: []const u8, - pos: usize, - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator, data: []const u8) Decoder { - return .{ - .data = data, - .pos = 0, - .allocator = allocator, - }; - } - - fn remaining(self: *Decoder) []const u8 { - return self.data[self.pos..]; - } - - fn readByte(self: *Decoder) !u8 { - if (self.pos >= self.data.len) return error.UnexpectedEof; - const b = self.data[self.pos]; - self.pos += 1; - return b; - } - - fn readBytes(self: *Decoder, n: usize) ![]const u8 { - if (self.pos + n > self.data.len) return error.UnexpectedEof; - const slice = self.data[self.pos .. self.pos + n]; - self.pos += n; - return slice; - } - - fn readArg(self: *Decoder, additional: u5) !u64 { - if (additional < 24) return additional; - switch (additional) { - 24 => return try self.readByte(), - 25 => { - const bytes = try self.readBytes(2); - return std.mem.bigToNative(u16, std.mem.bytesToValue(u16, bytes[0..2])); - }, - 26 => { - const bytes = try self.readBytes(4); - return std.mem.bigToNative(u32, std.mem.bytesToValue(u32, bytes[0..4])); - }, - 27 => { - const bytes = try self.readBytes(8); - return std.mem.bigToNative(u64, std.mem.bytesToValue(u64, bytes[0..8])); - }, - else => return error.InvalidValue, - } - } - - pub fn readTypeArg(self: *Decoder) !struct { major: MajorType, arg: u64 } { - const b = try self.readByte(); - const major: MajorType = @enumFromInt(@as(u3, @truncate(b >> 5))); - const additional: u5 = @truncate(b); - const arg = try self.readArg(additional); - return .{ .major = major, .arg = arg }; - } - - pub fn decodeUint(self: *Decoder) !u64 { - const ta = try self.readTypeArg(); - if (ta.major != .unsigned) return error.InvalidType; - return ta.arg; - } - - pub fn decodeInt(self: *Decoder) !i64 { - const ta = try self.readTypeArg(); - switch (ta.major) { - .unsigned => return @bitCast(ta.arg), - .negative => return -1 - @as(i64, @bitCast(ta.arg)), - else => return error.InvalidType, - } - } - - pub fn decodeText(self: *Decoder) ![]const u8 { - const ta = try self.readTypeArg(); - if (ta.major != .text) return error.InvalidType; - return try self.readBytes(@intCast(ta.arg)); - } - - pub fn decodeBytes(self: *Decoder) ![]const u8 { - const ta = try self.readTypeArg(); - if (ta.major != .bytes) return error.InvalidType; - return try self.readBytes(@intCast(ta.arg)); - } - - pub fn decodeArrayLen(self: *Decoder) !usize { - const ta = try self.readTypeArg(); - if (ta.major != .array) return error.InvalidType; - return @intCast(ta.arg); - } - - pub fn decodeMapLen(self: *Decoder) !usize { - const ta = try self.readTypeArg(); - if (ta.major != .map) return error.InvalidType; - return @intCast(ta.arg); - } - - pub fn decodeTag(self: *Decoder) !u64 { - const ta = try self.readTypeArg(); - if (ta.major != .tag) return error.InvalidType; - return ta.arg; - } - - pub fn decodeBool(self: *Decoder) !bool { - const b = try self.readByte(); - switch (b) { - 0xF4 => return false, - 0xF5 => return true, - else => return error.InvalidType, - } - } - - pub fn isNull(self: *Decoder) !bool { - if (self.pos >= self.data.len) return error.UnexpectedEof; - if (self.data[self.pos] == 0xF6) { - self.pos += 1; - return true; - } - return false; - } - - pub fn skip(self: *Decoder) !void { - const ta = try self.readTypeArg(); - switch (ta.major) { - .unsigned, .negative => {}, - .bytes, .text => { - _ = try self.readBytes(@intCast(ta.arg)); - }, - .array => { - var i: usize = 0; - while (i < ta.arg) : (i += 1) { - try self.skip(); - } - }, - .map => { - var i: usize = 0; - while (i < ta.arg) : (i += 1) { - try self.skip(); // key - try self.skip(); // value - } - }, - .tag => { - try self.skip(); - }, - .simple => { - // Handle floats - const additional: u5 = @truncate(self.data[self.pos - 1]); - switch (additional) { - 25 => _ = try self.readBytes(2), - 26 => _ = try self.readBytes(4), - 27 => _ = try self.readBytes(8), - else => {}, - } - }, - } - } -}; - -// ============================================================ -// Helper Functions -// ============================================================ - -// Encode a provenance payload -pub fn encodeProvenance( - allocator: std.mem.Allocator, - actor_id: []const u8, - actor_type: []const u8, - rationale: []const u8, - timestamp: []const u8, -) ![]u8 { - var encoder = Encoder.init(allocator); - errdefer encoder.deinit(); - - try encoder.encodeLithTag(.provenance); - try encoder.beginMap(3); - - // actor - try encoder.encodeText("actor"); - try encoder.encodeLithTag(.actor); - try encoder.beginMap(2); - try encoder.encodeText("id"); - try encoder.encodeText(actor_id); - try encoder.encodeText("type"); - try encoder.encodeText(actor_type); - - // rationale - try encoder.encodeText("rationale"); - try encoder.encodeText(rationale); - - // timestamp - try encoder.encodeText("timestamp"); - try encoder.encodeTag(0); // datetime tag - try encoder.encodeText(timestamp); - - const result = try allocator.dupe(u8, encoder.finish()); - encoder.deinit(); - return result; -} - -// Encode an error blob -pub fn encodeError( - allocator: std.mem.Allocator, - code: i32, - message: []const u8, -) ![]u8 { - var encoder = Encoder.init(allocator); - errdefer encoder.deinit(); - - try encoder.beginMap(2); - try encoder.encodeText("code"); - try encoder.encodeInt(code); - try encoder.encodeText("message"); - try encoder.encodeText(message); - - const result = try allocator.dupe(u8, encoder.finish()); - encoder.deinit(); - return result; -} - -// ============================================================ -// Tests -// ============================================================ - -test "encode unsigned integers" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.encodeUint(0); - try encoder.encodeUint(23); - try encoder.encodeUint(24); - try encoder.encodeUint(255); - try encoder.encodeUint(256); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0x00), result[0]); // 0 - try std.testing.expectEqual(@as(u8, 0x17), result[1]); // 23 - try std.testing.expectEqual(@as(u8, 0x18), result[2]); // 24 prefix - try std.testing.expectEqual(@as(u8, 0x18), result[3]); // 24 value - try std.testing.expectEqual(@as(u8, 0x18), result[4]); // 255 prefix - try std.testing.expectEqual(@as(u8, 0xFF), result[5]); // 255 value -} - -test "encode text string" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.encodeText("hello"); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0x65), result[0]); // text(5) - try std.testing.expectEqualStrings("hello", result[1..6]); -} - -test "encode simple map" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.beginMap(2); - try encoder.encodeText("name"); - try encoder.encodeText("Lith"); - try encoder.encodeText("version"); - try encoder.encodeUint(1); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0xA2), result[0]); // map(2) -} - -test "decode unsigned integer" { - const data = [_]u8{ 0x18, 0x64 }; // 100 - var decoder = Decoder.init(std.testing.allocator, &data); - - const value = try decoder.decodeUint(); - try std.testing.expectEqual(@as(u64, 100), value); -} - -test "decode text string" { - const data = [_]u8{ 0x65, 'h', 'e', 'l', 'l', 'o' }; - var decoder = Decoder.init(std.testing.allocator, &data); - - const text = try decoder.decodeText(); - try std.testing.expectEqualStrings("hello", text); -} - -test "encode provenance" { - const result = try encodeProvenance( - std.testing.allocator, - "user_123", - "human", - "Adding test data", - "2026-01-11T12:00:00Z", - ); - defer std.testing.allocator.free(result); - - // Verify it starts with provenance tag - try std.testing.expectEqual(@as(u8, 0xD9), result[0]); // tag (2-byte) -} diff --git a/lithoglyph/core-zig/src/crypto.zig b/lithoglyph/core-zig/src/crypto.zig deleted file mode 100644 index bb7ec157..00000000 --- a/lithoglyph/core-zig/src/crypto.zig +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph Cryptography Module -// -// Provides AES-256-GCM encryption for block payloads -// Designed to integrate with Svalinn Vault's existing crypto system -// -// NOTE: All crypto functions below are PLACEHOLDERS for structure only. -// Real implementation must use libsodium or std.crypto.aead.aes_gcm. - -const std = @import("std"); -const builtin = @import("builtin"); -const blocks = @import("blocks.zig"); - -// ============================================================ -// Constants -// ============================================================ - -pub const AES256_KEY_SIZE: usize = 32; // 256 bits -pub const AES_GCM_NONCE_SIZE: usize = 12; // 96 bits for GCM -pub const AES_GCM_TAG_SIZE: usize = 16; // 128 bits authentication tag - -// Encrypted flag bit in BlockHeader.flags -const ENCRYPTED_FLAG: u32 = 0x02; // bit 1 of BlockFlags - -// ============================================================ -// Error Types -// ============================================================ - -pub const CryptoError = error{ - InvalidKeySize, - EncryptionFailed, - DecryptionFailed, - AuthenticationFailed, - BufferTooSmall, -}; - -// ============================================================ -// AES-256-GCM Implementation -// ============================================================ - -// Encrypt block payload using AES-256-GCM -// Uses block_id as part of nonce for uniqueness -pub fn encryptBlockPayload( - block: *blocks.Block, - key: [AES256_KEY_SIZE]u8, -) !void { - if ((block.header.flags & ENCRYPTED_FLAG) != 0) { - return; // Already encrypted - } - - // Create nonce: block_id (8 bytes) + lower 4 bytes of sequence - var nonce: [AES_GCM_NONCE_SIZE]u8 = undefined; - std.mem.writeInt(u64, nonce[0..8], block.header.block_id, .little); - std.mem.writeInt(u32, nonce[8..12], @truncate(block.header.sequence), .little); - - // Encrypt payload in-place - const payload_len = block.header.payload_len; - if (payload_len == 0) { - return; // Nothing to encrypt - } - - // Make space for authentication tag at end - if (payload_len + AES_GCM_TAG_SIZE > blocks.PAYLOAD_SIZE) { - return CryptoError.BufferTooSmall; - } - - try aes256GcmEncryptInPlace( - block.payload[0..payload_len], - &key, - &nonce, - block.payload[payload_len .. payload_len + AES_GCM_TAG_SIZE], - ); - - // Update header - block.header.payload_len = @intCast(payload_len + AES_GCM_TAG_SIZE); - block.header.flags |= ENCRYPTED_FLAG; - - // Recalculate checksum of encrypted data - block.header.checksum = blocks.crc32c(&block.payload, block.header.payload_len); -} - -// Decrypt block payload using AES-256-GCM -pub fn decryptBlockPayload( - block: *blocks.Block, - key: [AES256_KEY_SIZE]u8, -) !void { - if ((block.header.flags & ENCRYPTED_FLAG) == 0) { - return; // Not encrypted - } - - const total_len = block.header.payload_len; - if (total_len < AES_GCM_TAG_SIZE) { - return CryptoError.AuthenticationFailed; - } - - const payload_len = total_len - AES_GCM_TAG_SIZE; - - // Create nonce (same as encryption) - var nonce: [AES_GCM_NONCE_SIZE]u8 = undefined; - std.mem.writeInt(u64, nonce[0..8], block.header.block_id, .little); - std.mem.writeInt(u32, nonce[8..12], @truncate(block.header.sequence), .little); - - try aes256GcmDecryptInPlace( - block.payload[0..payload_len], - &key, - &nonce, - block.payload[payload_len..total_len], - ); - - // Update header - block.header.payload_len = @intCast(payload_len); - block.header.flags &= ~ENCRYPTED_FLAG; - - // Recalculate checksum of decrypted data - block.header.checksum = blocks.crc32c(&block.payload, block.header.payload_len); -} - -// ============================================================ -// Journal Encryption -// ============================================================ - -// Journal entries need special handling because they contain -// both the operation and its inverse. -// NOTE: caller owns `out_buf` and must ensure it is at least -// `entry_data.len + AES_GCM_TAG_SIZE` bytes long. -pub fn encryptJournalEntry( - entry_data: []const u8, - key: [AES256_KEY_SIZE]u8, - entry_id: u64, - out_buf: []u8, -) !usize { - const needed = entry_data.len + AES_GCM_TAG_SIZE; - if (out_buf.len < needed) return CryptoError.BufferTooSmall; - - // Build nonce: entry_id (8 bytes) + "JNL\0" marker - var nonce: [AES_GCM_NONCE_SIZE]u8 = undefined; - std.mem.writeInt(u64, nonce[0..8], entry_id, .little); - nonce[8] = 'J'; - nonce[9] = 'N'; - nonce[10] = 'L'; - nonce[11] = 0; - - @memcpy(out_buf[0..entry_data.len], entry_data); - - try aes256GcmEncryptInPlace( - out_buf[0..entry_data.len], - &key, - &nonce, - out_buf[entry_data.len..needed], - ); - - return needed; -} - -// ============================================================ -// Key Derivation -// ============================================================ - -// Derive encryption key from master key and block type. -// PLACEHOLDER — real implementation must use HKDF or BLAKE3-KDF. -pub fn deriveBlockKey( - master_key: []const u8, - block_type: blocks.BlockType, - block_id: u64, -) ![AES256_KEY_SIZE]u8 { - var key: [AES256_KEY_SIZE]u8 = undefined; - const bt: u8 = @truncate(@intFromEnum(block_type)); - for (0..AES256_KEY_SIZE) |i| { - const shift: u6 = @intCast(i % 8); - const id_byte: u8 = @truncate(block_id >> shift); - key[i] = if (i < master_key.len) master_key[i] ^ bt ^ id_byte else bt ^ id_byte; - } - return key; -} - -// ============================================================ -// Placeholder Crypto Primitives -// (Replace with libsodium / std.crypto.aead.aes_gcm in production) -// ============================================================ - -fn aes256GcmEncryptInPlace( - data: []u8, - key: *const [AES256_KEY_SIZE]u8, - nonce: *const [AES_GCM_NONCE_SIZE]u8, - tag_out: []u8, -) !void { - _ = nonce; // placeholder — nonce not used in XOR stub - // INSECURE placeholder: XOR with key bytes only - for (data, 0..) |*byte, i| { - byte.* ^= key[i % AES256_KEY_SIZE]; - } - // Fill tag with dummy pattern - for (tag_out, 0..) |*b, i| { - b.* = @truncate(i); - } -} - -fn aes256GcmDecryptInPlace( - data: []u8, - key: *const [AES256_KEY_SIZE]u8, - nonce: *const [AES_GCM_NONCE_SIZE]u8, - tag: []const u8, -) !void { - _ = nonce; // placeholder - // Verify dummy tag - for (tag, 0..) |b, i| { - if (b != @as(u8, @truncate(i))) return CryptoError.AuthenticationFailed; - } - // INSECURE placeholder: XOR is its own inverse - for (data, 0..) |*byte, i| { - byte.* ^= key[i % AES256_KEY_SIZE]; - } -} diff --git a/lithoglyph/core-zig/src/crypto_test.zig b/lithoglyph/core-zig/src/crypto_test.zig deleted file mode 100644 index a950d5ac..00000000 --- a/lithoglyph/core-zig/src/crypto_test.zig +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Test suite for Lithoglyph cryptography module - -const std = @import("std"); -const crypto = @import("crypto.zig"); -const blocks = @import("blocks.zig"); - -test "block encryption/decryption roundtrip" { - // Create a test block - var block = blocks.Block.init(blocks.BlockType.document, 12345, 1); - - // Set some test data - const test_data = "Hello, Svalinn Vault! This is a secret credential."; - try block.setPayload(test_data); - - // Verify not encrypted initially - try std.testing.expect(!block.isEncrypted()); - - // Create encryption key - const key = [crypto.AES256_KEY_SIZE]u8{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - }; - - // Encrypt the block - try block.encrypt(key); - - // Verify encrypted - try std.testing.expect(block.isEncrypted()); - - // Verify payload changed (encrypted) - const encrypted_payload = block.getPayload(); - try std.testing.expect(!std.mem.eql(u8, test_data, encrypted_payload)); - - // Decrypt the block - try block.decrypt(key); - - // Verify decrypted - try std.testing.expect(!block.isEncrypted()); - - // Verify payload restored - const decrypted_payload = block.getPayload(); - try std.testing.expect(std.mem.eql(u8, test_data, decrypted_payload)); -} - -test "key derivation" { - const master_key = "master-secret-key-1234567890"; - const key1 = try blocks.Block.deriveKey(master_key, blocks.BlockType.document, 123); - const key2 = try blocks.Block.deriveKey(master_key, blocks.BlockType.document, 123); - - // Same parameters should give same key - try std.testing.expect(std.mem.eql(u8, &key1, &key2)); - - const key3 = try blocks.Block.deriveKey(master_key, blocks.BlockType.document, 456); - - // Different block ID should give different key - try std.testing.expect(!std.mem.eql(u8, &key1, &key3)); -} - -test "double encryption detection" { - var block = blocks.Block.init(blocks.BlockType.document, 999, 1); - const test_data = "test"; - try block.setPayload(test_data); - - const key = std.mem.zeroes([crypto.AES256_KEY_SIZE]u8); - - // First encryption - try block.encrypt(key); - try std.testing.expect(block.isEncrypted()); - - // Second encryption should be no-op - try block.encrypt(key); - try std.testing.expect(block.isEncrypted()); - - // Should still decrypt correctly - try block.decrypt(key); - try std.testing.expect(std.mem.eql(u8, test_data, block.getPayload())); -} diff --git a/lithoglyph/core-zig/src/types.zig b/lithoglyph/core-zig/src/types.zig deleted file mode 100644 index cf9ba0a0..00000000 --- a/lithoglyph/core-zig/src/types.zig +++ /dev/null @@ -1,264 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith.Bridge - Type Definitions -// -// Part of Lithoglyph: Stone-carved data for the ages. -// LgBlob = Lithoglyph Blob (abbreviated for C compatibility) - -const std = @import("std"); - -// ============================================================ -// Blob Encoding -// ============================================================ - -pub const BlobEncoding = enum(u8) { - cbor = 0, - cbor_compressed = 1, - reserved = 255, -}; - -// ============================================================ -// Blob Structure -// ============================================================ - -pub const LgBlob = extern struct { - data: ?[*]const u8, - len: usize, - encoding: BlobEncoding, - _padding: [7]u8 = [_]u8{0} ** 7, - - pub fn empty() LgBlob { - return .{ - .data = null, - .len = 0, - .encoding = .cbor, - }; - } - - pub fn fromSlice(slice: []const u8) LgBlob { - return .{ - .data = slice.ptr, - .len = slice.len, - .encoding = .cbor, - }; - } - - pub fn toSlice(self: LgBlob) ?[]const u8 { - if (self.data) |ptr| { - return ptr[0..self.len]; - } - return null; - } -}; - -// ============================================================ -// Status Codes -// ============================================================ - -pub const LithStatus = enum(i32) { - ok = 0, - - // Database errors (1xxx) - err_db_not_found = 1001, - err_db_already_open = 1002, - err_db_corrupted = 1003, - err_db_version_mismatch = 1004, - - // Transaction errors (2xxx) - err_txn_not_active = 2001, - err_txn_already_committed = 2002, - err_txn_already_aborted = 2003, - err_txn_conflict = 2004, - - // Document errors (3xxx) - err_doc_not_found = 3001, - err_doc_already_exists = 3002, - err_doc_validation_failed = 3003, - - // Collection errors (4xxx) - err_collection_not_found = 4001, - err_collection_already_exists = 4002, - - // Schema errors (5xxx) - err_schema_violation = 5001, - err_constraint_violation = 5002, - - // Internal errors (9xxx) - err_internal = 9001, - err_out_of_memory = 9002, - err_invalid_argument = 9003, - err_not_implemented = 9004, - - pub fn isOk(self: LithStatus) bool { - return self == .ok; - } - - pub fn isError(self: LithStatus) bool { - return @intFromEnum(self) > 0; - } -}; - -// ============================================================ -// Transaction Mode -// ============================================================ - -pub const LithTxnMode = enum(u8) { - read_only = 0, - read_write = 1, -}; - -// ============================================================ -// Operation Types -// ============================================================ - -pub const LithOpType = enum(u16) { - // Document operations - doc_insert = 0x0001, - doc_update = 0x0002, - doc_delete = 0x0003, - doc_replace = 0x0004, - - // Edge operations - edge_insert = 0x0010, - edge_delete = 0x0011, - edge_update = 0x0012, - - // Collection operations - collection_create = 0x0020, - collection_drop = 0x0021, - - // Schema operations - schema_create = 0x0030, - schema_alter = 0x0031, - - // Constraint operations - constraint_add = 0x0040, - constraint_drop = 0x0041, - - // Index operations - index_create = 0x0050, - index_drop = 0x0051, - - // Query operations - query_select = 0x0100, - query_aggregate = 0x0101, - query_explain = 0x0102, -}; - -// ============================================================ -// Block Types -// ============================================================ - -pub const LithBlockType = enum(u16) { - free = 0x0000, - superblock = 0x0001, - collection_meta = 0x0010, - document = 0x0011, - document_overflow = 0x0012, - edge_meta = 0x0020, - edge = 0x0021, - index_root = 0x0030, - index_internal = 0x0031, - index_leaf = 0x0032, - journal_segment = 0x0040, - schema = 0x0050, - constraint = 0x0051, - migration = 0x0060, -}; - -// ============================================================ -// CBOR Tags (Lith-specific) -// ============================================================ - -pub const CborTag = enum(u64) { - datetime = 0, - uri = 32, - self_described = 55799, - block_reference = 39001, - document_id = 39002, - collection_name = 39003, - provenance = 39004, - actor = 39005, - prompt_score = 39006, - functional_dependency = 39007, - proof = 39008, -}; - -// ============================================================ -// Render Options -// ============================================================ - -pub const LithRenderOpts = extern struct { - include_provenance: bool = true, - include_timestamps: bool = true, - pretty_print: bool = false, - max_depth: u32 = 10, - _padding: [3]u8 = [_]u8{0} ** 3, -}; - -// ============================================================ -// Result Structure -// ============================================================ - -pub const LithResult = extern struct { - result_blob: LgBlob, - provenance_blob: LgBlob, - status: LithStatus, - _padding: [4]u8 = [_]u8{0} ** 4, - err_blob: LgBlob, - - pub fn ok(result: LgBlob) LithResult { - return .{ - .result_blob = result, - .provenance_blob = LgBlob.empty(), - .status = .ok, - .err_blob = LgBlob.empty(), - }; - } - - pub fn okWithProvenance(result: LgBlob, provenance: LgBlob) LithResult { - return .{ - .result_blob = result, - .provenance_blob = provenance, - .status = .ok, - .err_blob = LgBlob.empty(), - }; - } - - pub fn err(status: LithStatus, err_blob: LgBlob) LithResult { - return .{ - .result_blob = LgBlob.empty(), - .provenance_blob = LgBlob.empty(), - .status = status, - .err_blob = err_blob, - }; - } -}; - -// ============================================================ -// Tests -// ============================================================ - -test "LgBlob empty" { - const blob = LgBlob.empty(); - try std.testing.expectEqual(@as(?[*]const u8, null), blob.data); - try std.testing.expectEqual(@as(usize, 0), blob.len); -} - -test "LgBlob fromSlice" { - const data = "test data"; - const blob = LgBlob.fromSlice(data); - try std.testing.expectEqual(@as(usize, 9), blob.len); - - if (blob.toSlice()) |slice| { - try std.testing.expectEqualStrings("test data", slice); - } else { - try std.testing.expect(false); - } -} - -test "LithStatus" { - try std.testing.expect(LithStatus.ok.isOk()); - try std.testing.expect(!LithStatus.ok.isError()); - try std.testing.expect(!LithStatus.err_doc_not_found.isOk()); - try std.testing.expect(LithStatus.err_doc_not_found.isError()); -} diff --git a/lithoglyph/core-zig/test-db-open.c b/lithoglyph/core-zig/test-db-open.c deleted file mode 100644 index b0b6fa54..00000000 --- a/lithoglyph/core-zig/test-db-open.c +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Test database open - -#include -#include -#include -#include - -typedef struct { - const uint8_t* ptr; - size_t len; -} LgBlob; - -extern int lith_version(void); -extern int lith_db_open(const uint8_t* path, size_t path_len, const uint8_t* opts, size_t opts_len, void** out_db, LgBlob* out_err); -extern void lith_db_close(void* db); - -int main() { - printf("Version: %d\n", lith_version()); - - const char* path = "test-simple.lgh"; - void* db = NULL; - LgBlob err = {.ptr = NULL, .len = 0}; - - printf("Opening database: %s\n", path); - printf(" path ptr: %p\n", (void*)path); - printf(" path len: %zu\n", strlen(path)); - printf(" db ptr address: %p\n", (void*)&db); - printf(" err ptr address: %p\n", (void*)&err); - - int status = lith_db_open((const uint8_t*)path, strlen(path), NULL, 0, &db, &err); - - printf("Status: %d\n", status); - printf("DB handle: %p\n", db); - printf("Error ptr: %p, len: %zu\n", (void*)err.ptr, err.len); - - if (err.ptr && err.len > 0) { - printf("Error: %.*s\n", (int)err.len, err.ptr); - } - - if (db) { - printf("Closing database\n"); - lith_db_close(db); - } - - return status; -} diff --git a/lithoglyph/core-zig/test-execution.sh b/lithoglyph/core-zig/test-execution.sh deleted file mode 100755 index 4ca81b1b..00000000 --- a/lithoglyph/core-zig/test-execution.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph Execution Test -# -# Verifies that the block storage layer works correctly end-to-end. -# Tests database creation, block allocation, writes, reads, and journal. - -set -euo pipefail - -echo "=== Lithoglyph Execution Test ===" -echo "" - -# Create test program -cat > test_execution.zig << 'EOF' -const std = @import("std"); -const blocks = @import("src/blocks.zig"); - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const path = "execution_test.lgh"; - defer std.fs.cwd().deleteFile(path) catch {}; - - std.debug.print("Creating database: {s}\n", .{path}); - const storage = try blocks.BlockStorage.open(allocator, path); - defer storage.deinit(); - - std.debug.print("✅ Database created\n", .{}); - std.debug.print(" Superblock version: {d}\n", .{storage.superblock.version}); - std.debug.print(" Block count: {d}\n", .{storage.superblock.block_count}); - - // Allocate document blocks - std.debug.print("\nAllocating 5 document blocks...\n", .{}); - var i: usize = 0; - var block_ids: [5]u64 = undefined; - while (i < 5) : (i += 1) { - const block_id = try storage.allocateBlock(.document); - block_ids[i] = block_id; - - var block = try storage.readBlock(block_id); - - var buf: [100]u8 = undefined; - const content = try std.fmt.bufPrint(&buf, "Document {d} content", .{i + 1}); - try block.setPayload(content); - - try storage.writeBlock(block_id, &block); - std.debug.print(" Block {d}: {s}\n", .{block_id, content}); - } - - std.debug.print("✅ Documents written\n", .{}); - - // Read blocks back - std.debug.print("\nReading blocks back...\n", .{}); - i = 0; - while (i < 5) : (i += 1) { - const block = try storage.readBlock(block_ids[i]); - const payload = block.getPayload(); - std.debug.print(" Block {d}: {s}\n", .{block_ids[i], payload}); - } - - std.debug.print("✅ Documents read\n", .{}); - - // Append journal entries - std.debug.print("\nAppending journal entries...\n", .{}); - const j1 = try storage.appendJournal("Operation: created documents"); - const j2 = try storage.appendJournal("Operation: verified reads"); - std.debug.print(" Journal entry 1: block {d}\n", .{j1}); - std.debug.print(" Journal entry 2: block {d}\n", .{j2}); - - // Verify journal linkage - const j2_block = try storage.readBlock(j2); - if (j2_block.header.prev_block_id == j1) { - std.debug.print("✅ Journal entries linked correctly\n", .{}); - } else { - std.debug.print("❌ Journal linkage broken\n", .{}); - return error.JournalLinkageError; - } - - // Final stats - std.debug.print("\nFinal database state:\n", .{}); - std.debug.print(" Total blocks: {d}\n", .{storage.superblock.block_count}); - std.debug.print(" Journal head: {d}\n", .{storage.superblock.journal_head}); - std.debug.print(" Journal tail: {d}\n", .{storage.superblock.journal_tail}); - - std.debug.print("\n✅ All execution tests passed!\n", .{}); -} -EOF - -echo "=== Compiling test program ===" -zig build-exe test_execution.zig -O ReleaseSafe -if [ ! -f "test_execution" ] && [ ! -f "test_execution.exe" ]; then - echo "❌ Compilation failed" - exit 1 -fi -echo "✅ Test program compiled" -echo "" - -echo "=== Running execution test ===" -if [ -f "test_execution" ]; then - ./test_execution - EXIT_CODE=$? -elif [ -f "test_execution.exe" ]; then - ./test_execution.exe - EXIT_CODE=$? -else - echo "❌ Test executable not found" - exit 1 -fi - -if [ $EXIT_CODE -eq 0 ]; then - echo "" - echo "=========================================" - echo "✅ Execution test passed!" - echo "=========================================" -else - echo "" - echo "❌ Execution test failed with code $EXIT_CODE" - exit 1 -fi - -# Cleanup -echo "" -echo "=== Cleanup ===" -rm -f test_execution test_execution.exe test_execution.zig -rm -f test_execution.o test_execution.obj -rm -f execution_test.lgh -rm -f *.lgh -echo "✅ Cleanup complete" diff --git a/lithoglyph/core-zig/test-ffi-integration.c b/lithoglyph/core-zig/test-ffi-integration.c deleted file mode 100644 index 073eb400..00000000 --- a/lithoglyph/core-zig/test-ffi-integration.c +++ /dev/null @@ -1,575 +0,0 @@ -/* SPDX-License-Identifier: MPL-2.0 */ -/* Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) */ -/* - * FFI Integration Tests — Tests the Zig bridge from C - * - * Exercises the complete FFI surface defined in generated/abi/bridge.h, - * simulating what Factor/Forth/BEAM runtimes do when calling the bridge. - * - * Compile: - * cc -I../generated/abi -o test-ffi test-ffi-integration.c -L. -lbridge - */ - -#include -#include -#include - -#include "bridge.h" - -/* ============================================================ - * Test Helpers - * ============================================================ */ - -static int test_count = 0; -static int pass_count = 0; -static int fail_count = 0; - -#define RUN_TEST(fn) do { \ - test_count++; \ - printf("=== Test %d: %s ===\n", test_count, #fn); \ - if (fn() == 0) { \ - pass_count++; \ - printf("PASS: %s\n\n", #fn); \ - } else { \ - fail_count++; \ - printf("FAIL: %s\n\n", #fn); \ - } \ -} while(0) - -static void print_blob(const char* label, const LgBlob* blob) { - printf(" %s: ", label); - if (blob->ptr && blob->len > 0) { - printf("%.*s\n", (int)blob->len, (const char*)blob->ptr); - } else { - printf("(empty)\n"); - } -} - -static void free_blob(LgBlob* blob) { - if (blob->ptr) { - lith_blob_free(blob); - } -} - -/* Open a test database, return 0 on success */ -static int open_test_db(const char* name, LithDb** db, LgBlob* err) { - LithStatus s = lith_db_open( - (const uint8_t*)name, strlen(name), - NULL, 0, - db, err - ); - if (s != LITH_OK) { - print_blob("open error", err); - free_blob(err); - return 1; - } - return 0; -} - -/* ============================================================ - * Test 1: Version - * ============================================================ */ -static int test_version(void) { - uint32_t v = lith_version(); - printf(" version = %u (expected 100 = 0.1.0)\n", v); - return (v == 100) ? 0 : 1; -} - -/* ============================================================ - * Test 2: Database Lifecycle (open + close) - * ============================================================ */ -static int test_database_lifecycle(void) { - LithDb* db = NULL; - LgBlob err = {0}; - - if (open_test_db("test-ffi.lgh", &db, &err)) return 1; - printf(" db handle: %p\n", (void*)db); - - LithStatus s = lith_db_close(db); - return (s == LITH_OK) ? 0 : 1; -} - -/* ============================================================ - * Test 3: Transaction begin + commit (empty) - * ============================================================ */ -static int test_transactions(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-txn.lgh", &db, &err)) return 1; - - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { print_blob("begin error", &err); free_blob(&err); lith_db_close(db); return 1; } - printf(" txn handle: %p\n", (void*)txn); - - s = lith_txn_commit(txn, &err); - if (s != LITH_OK) { print_blob("commit error", &err); free_blob(&err); lith_db_close(db); return 1; } - - lith_db_close(db); - return 0; -} - -/* ============================================================ - * Test 4: Transaction abort - * ============================================================ */ -static int test_txn_abort(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-abort.lgh", &db, &err)) return 1; - - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - /* Apply something then abort — should not persist */ - const char* op = "{\"op\":\"insert\",\"doc\":{\"tmp\":true}}"; - LgResult r = lith_apply(txn, (const uint8_t*)op, strlen(op)); - printf(" apply status before abort: %d\n", r.status); - free_blob(&r.data); - free_blob(&r.error_blob); - - s = lith_txn_abort(txn); - printf(" abort status: %d\n", s); - - lith_db_close(db); - return (s == LITH_OK) ? 0 : 1; -} - -/* ============================================================ - * Test 5: Apply operation (read-write, buffered) - * ============================================================ */ -static int test_apply_readwrite(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-apply-rw.lgh", &db, &err)) return 1; - - /* Must use read-write mode for apply */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* op = "{\"op\":\"insert\",\"collection\":\"users\",\"doc\":{\"name\":\"Alice\"}}"; - LgResult result = lith_apply(txn, (const uint8_t*)op, strlen(op)); - - printf(" result status: %d (expected 0 = OK)\n", result.status); - print_blob("result data", &result.data); - - int ok = (result.status == LITH_OK); - free_blob(&result.data); - free_blob(&result.error_blob); - - s = lith_txn_commit(txn, &err); - free_blob(&err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 6: Apply + commit + read_blocks (round-trip) - * ============================================================ */ -static int test_apply_commit_readback(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-roundtrip.lgh", &db, &err)) return 1; - - /* Insert a document */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc = "{\"name\":\"Bob\",\"age\":30}"; - LgResult r = lith_apply(txn, (const uint8_t*)doc, strlen(doc)); - free_blob(&r.data); - free_blob(&r.error_blob); - - s = lith_txn_commit(txn, &err); - free_blob(&err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - /* Read back all document blocks */ - LgBlob data = {0}; - LgBlob read_err = {0}; - s = lith_read_blocks(db, LG_BLOCK_TYPE_DOCUMENT, &data, &read_err); - printf(" read_blocks status: %d\n", s); - print_blob("blocks", &data); - - int ok = (s == LITH_OK && data.ptr != NULL && data.len > 2); /* more than "[]" */ - free_blob(&data); - free_blob(&read_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 7: Update block - * ============================================================ */ -static int test_update_block(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-update.lgh", &db, &err)) return 1; - - /* Insert first */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc1 = "{\"version\":1}"; - LgResult r = lith_apply(txn, (const uint8_t*)doc1, strlen(doc1)); - /* Parse block_id from result — for simplicity, use block_id=1 (first allocation) */ - free_blob(&r.data); - free_blob(&r.error_blob); - - s = lith_txn_commit(txn, &err); - free_blob(&err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - /* Update the block */ - s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc2 = "{\"version\":2}"; - LgBlob update_err = {0}; - s = lith_update_block(txn, 1, (const uint8_t*)doc2, strlen(doc2), &update_err); - printf(" update_block status: %d\n", s); - - int ok = (s == LITH_OK); - free_blob(&update_err); - - LgBlob commit_err = {0}; - lith_txn_commit(txn, &commit_err); - free_blob(&commit_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 8: Delete block - * ============================================================ */ -static int test_delete_block(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-delete.lgh", &db, &err)) return 1; - - /* Insert a block */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc = "{\"delete_me\":true}"; - LgResult r = lith_apply(txn, (const uint8_t*)doc, strlen(doc)); - free_blob(&r.data); - free_blob(&r.error_blob); - lith_txn_commit(txn, &err); - free_blob(&err); - - /* Delete the block */ - s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - LgBlob del_err = {0}; - s = lith_delete_block(txn, 1, &del_err); - printf(" delete_block status: %d\n", s); - - int ok = (s == LITH_OK); - free_blob(&del_err); - - LgBlob commit_err = {0}; - lith_txn_commit(txn, &commit_err); - free_blob(&commit_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 9: Read blocks by type - * ============================================================ */ -static int test_read_blocks_by_type(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-read-type.lgh", &db, &err)) return 1; - - /* Insert some documents */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* docs[] = { - "{\"item\":\"alpha\"}", - "{\"item\":\"beta\"}", - "{\"item\":\"gamma\"}", - }; - for (int i = 0; i < 3; i++) { - LgResult r = lith_apply(txn, (const uint8_t*)docs[i], strlen(docs[i])); - free_blob(&r.data); - free_blob(&r.error_blob); - } - lith_txn_commit(txn, &err); - free_blob(&err); - - /* Read by document type */ - LgBlob data = {0}; - LgBlob read_err = {0}; - s = lith_read_blocks(db, LG_BLOCK_TYPE_DOCUMENT, &data, &read_err); - printf(" read_blocks (type 0x0011) status: %d\n", s); - print_blob("blocks", &data); - - int ok = (s == LITH_OK && data.ptr != NULL); - free_blob(&data); - free_blob(&read_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 10: Render block - * ============================================================ */ -static int test_render_block(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-render-block.lgh", &db, &err)) return 1; - - /* Insert a document */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc = "{\"rendered\":true}"; - LgResult r = lith_apply(txn, (const uint8_t*)doc, strlen(doc)); - free_blob(&r.data); - free_blob(&r.error_blob); - lith_txn_commit(txn, &err); - free_blob(&err); - - /* Render block 1 */ - LgBlob text = {0}; - LgBlob render_err = {0}; - LgRenderOpts opts = { .format = 0, .include_metadata = false }; - s = lith_render_block(db, 1, opts, &text, &render_err); - printf(" render_block status: %d\n", s); - print_blob("rendered", &text); - - int ok = (s == LITH_OK && text.ptr != NULL); - free_blob(&text); - free_blob(&render_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 11: Render journal - * ============================================================ */ -static int test_render_journal(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-render-journal.lgh", &db, &err)) return 1; - - /* Insert something to generate journal entries */ - LithStatus s = lith_txn_begin(db, LG_TXN_READ_WRITE, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* doc = "{\"journaled\":true}"; - LgResult r = lith_apply(txn, (const uint8_t*)doc, strlen(doc)); - free_blob(&r.data); - free_blob(&r.error_blob); - lith_txn_commit(txn, &err); - free_blob(&err); - - /* Render journal since sequence 0 */ - LgBlob text = {0}; - LgBlob journal_err = {0}; - LgRenderOpts opts = { .format = 0, .include_metadata = false }; - s = lith_render_journal(db, 0, opts, &text, &journal_err); - printf(" render_journal status: %d\n", s); - print_blob("journal", &text); - - int ok = (s == LITH_OK && text.ptr != NULL); - free_blob(&text); - free_blob(&journal_err); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Test 12: Introspection (schema + constraints) - * ============================================================ */ -static int test_introspection(void) { - LithDb* db = NULL; - LgBlob err = {0}; - - if (open_test_db("test-intro.lgh", &db, &err)) return 1; - - /* Schema */ - LgBlob schema = {0}; - LithStatus s = lith_introspect_schema(db, &schema, &err); - printf(" schema status: %d\n", s); - print_blob("schema", &schema); - free_blob(&schema); - free_blob(&err); - - if (s != LITH_OK) { lith_db_close(db); return 1; } - - /* Constraints */ - LgBlob constraints = {0}; - LgBlob c_err = {0}; - s = lith_introspect_constraints(db, &constraints, &c_err); - printf(" constraints status: %d\n", s); - print_blob("constraints", &constraints); - free_blob(&constraints); - free_blob(&c_err); - - lith_db_close(db); - return (s == LITH_OK) ? 0 : 1; -} - -/* ============================================================ - * Test 13: Proof init builtins - * ============================================================ */ -static int test_proof_init_builtins(void) { - LithStatus s = lith_proof_init_builtins(); - printf(" init_builtins status: %d\n", s); - return (s == LITH_OK) ? 0 : 1; -} - -/* ============================================================ - * Test 14: Proof register + unregister verifier - * ============================================================ */ -static LithStatus dummy_verifier(const uint8_t* proof, size_t len, void* ctx) { - (void)proof; (void)len; (void)ctx; - return LITH_OK; -} - -static int test_proof_register_unregister(void) { - const char* type_name = "test-verifier"; - LithStatus s = lith_proof_register_verifier( - (const uint8_t*)type_name, strlen(type_name), - dummy_verifier, NULL - ); - printf(" register status: %d\n", s); - if (s != LITH_OK) return 1; - - s = lith_proof_unregister_verifier( - (const uint8_t*)type_name, strlen(type_name) - ); - printf(" unregister status: %d\n", s); - if (s != LITH_OK) return 1; - - /* Unregister again should fail with NOT_FOUND */ - s = lith_proof_unregister_verifier( - (const uint8_t*)type_name, strlen(type_name) - ); - printf(" double-unregister status: %d (expected %d = NOT_FOUND)\n", s, LITH_ERR_NOT_FOUND); - return (s == LITH_ERR_NOT_FOUND) ? 0 : 1; -} - -/* ============================================================ - * Test 15: Proof verify - * ============================================================ */ -static int test_proof_verify(void) { - /* Ensure builtins are registered */ - lith_proof_init_builtins(); - - const char* proof_json = "{\"type\":\"fd-holds\",\"data\":\"dGVzdA==\"}"; - bool valid = false; - LgBlob err = {0}; - - LithStatus s = lith_proof_verify( - (const uint8_t*)proof_json, strlen(proof_json), - &valid, &err - ); - printf(" verify status: %d, valid: %s\n", s, valid ? "true" : "false"); - free_blob(&err); - - return (s == LITH_OK && valid) ? 0 : 1; -} - -/* ============================================================ - * Test 16: Blob free on NULL (null safety) - * ============================================================ */ -static int test_blob_free_null(void) { - LgBlob empty = { .ptr = NULL, .len = 0 }; - /* Should not crash */ - lith_blob_free(&empty); - printf(" blob_free(NULL) did not crash\n"); - return 0; -} - -/* ============================================================ - * Test 17: Apply on read-only transaction (should fail) - * ============================================================ */ -static int test_apply_readonly_rejected(void) { - LithDb* db = NULL; - LithTxn* txn = NULL; - LgBlob err = {0}; - - if (open_test_db("test-ro.lgh", &db, &err)) return 1; - - LithStatus s = lith_txn_begin(db, LG_TXN_READ_ONLY, &txn, &err); - if (s != LITH_OK) { lith_db_close(db); return 1; } - - const char* op = "{\"op\":\"insert\",\"doc\":{\"x\":1}}"; - LgResult result = lith_apply(txn, (const uint8_t*)op, strlen(op)); - - printf(" apply on read-only status: %d (expected non-zero)\n", result.status); - int ok = (result.status != LITH_OK); /* should be rejected */ - - free_blob(&result.data); - free_blob(&result.error_blob); - lith_txn_abort(txn); - lith_db_close(db); - - return ok ? 0 : 1; -} - -/* ============================================================ - * Main - * ============================================================ */ -int main(void) { - printf("======================================\n"); - printf("Lithoglyph FFI Integration Tests\n"); - printf("(using generated/abi/bridge.h)\n"); - printf("======================================\n\n"); - - RUN_TEST(test_version); - RUN_TEST(test_database_lifecycle); - RUN_TEST(test_transactions); - RUN_TEST(test_txn_abort); - RUN_TEST(test_apply_readwrite); - RUN_TEST(test_apply_commit_readback); - RUN_TEST(test_update_block); - RUN_TEST(test_delete_block); - RUN_TEST(test_read_blocks_by_type); - RUN_TEST(test_render_block); - RUN_TEST(test_render_journal); - RUN_TEST(test_introspection); - RUN_TEST(test_proof_init_builtins); - RUN_TEST(test_proof_register_unregister); - RUN_TEST(test_proof_verify); - RUN_TEST(test_blob_free_null); - RUN_TEST(test_apply_readonly_rejected); - - printf("======================================\n"); - printf("Results: %d/%d passed", pass_count, test_count); - if (fail_count > 0) { - printf(" (%d FAILED)", fail_count); - } - printf("\n======================================\n"); - - return fail_count; -} diff --git a/lithoglyph/core-zig/test-install.sh b/lithoglyph/core-zig/test-install.sh deleted file mode 100755 index 1a2e9512..00000000 --- a/lithoglyph/core-zig/test-install.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph Installation Test -# -# Verifies that the Zig bridge can be built and installed correctly. -# Tests both static and shared library builds. - -set -euo pipefail - -echo "=== Lithoglyph Installation Test ===" -echo "" - -# Check Zig installation -if ! command -v zig &> /dev/null; then - echo "❌ Zig not found. Please install Zig 0.13+ first." - exit 1 -fi - -ZIG_VERSION=$(zig version) -echo "✅ Zig version: $ZIG_VERSION" -echo "" - -# Build static library -echo "=== Building static library ===" -zig build-lib src/bridge.zig -O ReleaseSafe -if [ -f "libbridge.a" ]; then - SIZE=$(du -h libbridge.a | cut -f1) - echo "✅ Static library built: libbridge.a ($SIZE)" -else - echo "❌ Static library build failed" - exit 1 -fi -echo "" - -# Build shared library -echo "=== Building shared library ===" -zig build-lib -dynamic src/bridge.zig -O ReleaseSafe -if [ -f "libbridge.so" ] || [ -f "libbridge.dylib" ] || [ -f "bridge.dll" ]; then - if [ -f "libbridge.so" ]; then - SIZE=$(du -h libbridge.so | cut -f1) - echo "✅ Shared library built: libbridge.so ($SIZE)" - elif [ -f "libbridge.dylib" ]; then - SIZE=$(du -h libbridge.dylib | cut -f1) - echo "✅ Shared library built: libbridge.dylib ($SIZE)" - else - SIZE=$(du -h bridge.dll | cut -f1) - echo "✅ Shared library built: bridge.dll ($SIZE)" - fi -else - echo "❌ Shared library build failed" - exit 1 -fi -echo "" - -# Verify ABI exports -echo "=== Verifying ABI exports ===" -if [ -f "libbridge.so" ]; then - EXPORTS=$(nm -D libbridge.so | grep "lith_" | wc -l) - echo "Found $EXPORTS exported lith_* functions" - - if nm -D libbridge.so | grep -q "lith_db_open"; then - echo "✅ lith_db_open found" - else - echo "❌ lith_db_open missing" - exit 1 - fi - - if nm -D libbridge.so | grep -q "lith_apply"; then - echo "✅ lith_apply found" - else - echo "❌ lith_apply missing" - exit 1 - fi - - if nm -D libbridge.so | grep -q "lith_txn_begin"; then - echo "✅ lith_txn_begin found" - else - echo "❌ lith_txn_begin missing" - exit 1 - fi -else - echo "ℹ️ Skipping ABI verification (not on Linux)" -fi -echo "" - -# Test block storage module -echo "=== Testing block storage ===" -zig test src/blocks.zig -echo "✅ Block storage tests passed" -echo "" - -# Test bridge module -echo "=== Testing bridge ===" -zig test src/bridge.zig -echo "✅ Bridge tests passed" -echo "" - -# Cleanup -echo "=== Cleanup ===" -rm -f libbridge.a libbridge.so libbridge.dylib bridge.dll -rm -f *.o *.obj -rm -f test*.lgh -echo "✅ Cleanup complete" -echo "" - -echo "=========================================" -echo "✅ All installation tests passed!" -echo "=========================================" diff --git a/lithoglyph/core-zig/test-version-only.c b/lithoglyph/core-zig/test-version-only.c deleted file mode 100644 index e3f99c56..00000000 --- a/lithoglyph/core-zig/test-version-only.c +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Minimal test - just call lith_version - -#include - -extern int lith_version(void); - -int main() { - printf("Calling lith_version...\n"); - int version = lith_version(); - printf("Version: %d\n", version); - return 0; -} diff --git a/lithoglyph/core-zig/test_install_blocks.sh b/lithoglyph/core-zig/test_install_blocks.sh deleted file mode 100755 index 8c71fbde..00000000 --- a/lithoglyph/core-zig/test_install_blocks.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# Lithoglyph Block Storage Installation Test - -set -euo pipefail - -echo "=== Lithoglyph Block Storage Installation Test ===" -echo "" - -# Check Zig -if ! command -v zig &> /dev/null; then - echo "❌ Zig not found" - exit 1 -fi - -echo "✅ Zig version: $(zig version)" -echo "" - -# Test block storage -echo "=== Testing block storage module ===" -zig test src/blocks.zig -echo "✅ All 9 tests passed" -echo "" - -echo "=========================================" -echo "✅ Installation test passed!" -echo "=========================================" diff --git a/lithoglyph/distributed/README.md b/lithoglyph/distributed/README.md deleted file mode 100644 index 088f50a0..00000000 --- a/lithoglyph/distributed/README.md +++ /dev/null @@ -1,245 +0,0 @@ -# Lith Distributed Mode - -Distributed computing features for Lith including cluster coordination, consensus, sharding, and replication. - -## Features - -| Feature | File | Description | -|---------|------|-------------| -| Cluster | `Lith_Distributed_Cluster.res` | Node discovery and membership | -| Consensus | `Lith_Distributed_Consensus.res` | Raft-based consensus | -| Sharding | `Lith_Distributed_Sharding.res` | Data partitioning | -| Replication | `Lith_Distributed_Replication.res` | Data synchronization | - -## Cluster Coordination - -Node discovery, membership, and cluster state management. - -```rescript -// Create cluster state -let cluster = makeState(~clusterId="lith-cluster") - -// Add node to cluster -addNode(cluster, { - id: "node-1", - address: "192.168.1.10", - port: 7946, - status: Active, - role: Follower, - lastHeartbeat: Js.Date.now(), - metadata: Js.Dict.empty(), -}) - -// Check if node is leader -if isLeader(cluster, "node-1") { - // Handle leader responsibilities -} - -// Get active nodes -let activeNodes = getActiveNodes(cluster) -``` - -### Node Roles - -| Role | Description | -|------|-------------| -| Leader | Coordinates writes, manages cluster | -| Follower | Replicates data, handles reads | -| Candidate | Participating in leader election | - -### Node Status - -| Status | Description | -|--------|-------------| -| Starting | Node is initializing | -| Joining | Node is joining cluster | -| Active | Node is operational | -| Leaving | Node is gracefully leaving | -| Down | Node is unreachable | - -## Consensus (Raft) - -Raft-based distributed consensus for leader election and log replication. - -```rescript -// Create consensus node -let node = make(~nodeId="node-1", ~electionTimeout=5000.0) - -// Handle vote request from candidate -let response = handleVoteRequest(node, { - term: 2, - candidateId: "node-2", - lastLogIndex: 10, - lastLogTerm: 1, -}) - -// Start election if timeout elapsed -if electionTimeoutElapsed(node) { - let request = startElection(node) - // Send vote requests to peers -} - -// Append command (leader only) -switch appendCommand(node, Js.Json.string("command")) { -| Some(entry) => // Command logged -| None => // Not leader -} -``` - -### Raft States - -| State | Description | -|-------|-------------| -| Follower | Receives heartbeats, responds to requests | -| Candidate | Requesting votes for leader election | -| Leader | Manages log replication, sends heartbeats | - -## Sharding - -Consistent hashing and data partitioning. - -```rescript -// Create shard manager -let manager = make(~config={ - strategy: Hash, - numShards: 16, - replicationFactor: 3, - virtualNodes: 150, -}) - -// Initialize with nodes -initializeShards(manager, ["node-1", "node-2", "node-3"]) - -// Get shard for key -switch getShardForKey(manager, "user:12345") { -| Some(shard) => - Console.log(`Key maps to shard ${Int.toString(shard.id)}`) - Console.log(`Primary: ${shard.primaryNode}`) -| None => () -} - -// Get node using consistent hashing -switch getNodeForKey(manager, "user:12345") { -| Some(nodeId) => // Route request to node -| None => () -} -``` - -### Sharding Strategies - -| Strategy | Description | -|----------|-------------| -| Hash | Consistent hashing with virtual nodes | -| Range | Key range-based partitioning | -| Directory | Lookup table-based routing | - -## Replication - -Data synchronization with configurable consistency levels. - -```rescript -// Create replication manager -let repl = make(~nodeId="node-1", ~config={ - mode: SemiSync, - readConsistency: One, - writeConsistency: Quorum, - replicationFactor: 3, - syncIntervalMs: 100.0, - conflictResolution: "last-write-wins", -}) - -// Create and queue replication event -let event = createEvent( - repl, - Write, - "users", - "user-123", - Some(Js.Json.string("{...}")), -) -queueEvent(repl, event) - -// Check if write succeeded -if isWriteSuccessful(repl, ackCount) { - // Commit write -} - -// Get replication lag -let stats = getLagStats(repl) -Console.log(`Max lag: ${Int.toString(stats.maxLag)} events`) -``` - -### Consistency Levels - -| Level | Description | Use Case | -|-------|-------------|----------| -| One | Single node responds | Fast reads | -| Quorum | Majority responds | Balanced | -| All | All nodes respond | Strong consistency | -| LocalQuorum | Local DC quorum | Multi-region | - -### Replication Modes - -| Mode | Description | Latency | Durability | -|------|-------------|---------|------------| -| Synchronous | Wait for all replicas | High | High | -| Asynchronous | Fire and forget | Low | Lower | -| SemiSync | Wait for quorum | Medium | Good | - -## Architecture - -``` -distributed/ -├── README.md -└── src/ - ├── Lith_Distributed_Cluster.res # Cluster coordination - ├── Lith_Distributed_Consensus.res # Raft consensus - ├── Lith_Distributed_Sharding.res # Data sharding - └── Lith_Distributed_Replication.res # Data replication -``` - -## Deployment Patterns - -### Single Region - -``` -┌─────────────────────────────────────────┐ -│ Load Balancer │ -└────────────┬───────────┬────────────────┘ - │ │ - ┌──────▼──┐ ┌────▼────┐ ┌────────┐ - │ Node 1 │ │ Node 2 │ │ Node 3 │ - │ (Leader)│ │(Follower│ │(Follower│ - └─────────┘ └─────────┘ └─────────┘ -``` - -### Multi-Region - -``` -Region A Region B -┌─────────────┐ ┌─────────────┐ -│ Node 1 │◄──────────►│ Node 4 │ -│ Node 2 │ │ Node 5 │ -│ Node 3 │ │ Node 6 │ -└─────────────┘ └─────────────┘ -``` - -## Best Practices - -### Cluster Sizing -- Minimum 3 nodes for fault tolerance -- Use odd number of nodes for consensus -- Scale shards based on data volume - -### Consistency Tuning -- Use `One` for read-heavy workloads -- Use `Quorum` for balanced reads/writes -- Use `All` only when strong consistency required - -### Replication Factor -- 3 for most workloads -- 5 for critical data -- Consider network bandwidth - -## License - -PMPL-1.0-or-later diff --git a/lithoglyph/distributed/rescript.json b/lithoglyph/distributed/rescript.json deleted file mode 100644 index df4b7674..00000000 --- a/lithoglyph/distributed/rescript.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "@lith/distributed", - "sources": [{ "dir": "src" }], - "package-specs": [{ "module": "es6", "in-source": true }], - "suffix": ".res.js", - "bs-dependencies": ["@rescript/core"], - "bsc-flags": ["-open RescriptCore"] -} diff --git a/lithoglyph/distributed/src/Lith_Distributed_Cluster.res b/lithoglyph/distributed/src/Lith_Distributed_Cluster.res deleted file mode 100644 index 7609f8ee..00000000 --- a/lithoglyph/distributed/src/Lith_Distributed_Cluster.res +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Cluster Coordination - * - * Node discovery, membership, and cluster state management - */ - -/** Node status */ -type nodeStatus = - | Starting - | Joining - | Active - | Leaving - | Down - -/** Node role */ -type nodeRole = - | Leader - | Follower - | Candidate - -/** Node information */ -type nodeInfo = { - id: string, - address: string, - port: int, - status: nodeStatus, - role: nodeRole, - lastHeartbeat: float, - metadata: Js.Dict.t, -} - -/** Cluster state */ -type clusterState = { - clusterId: string, - mutable nodes: array, - mutable leader: option, - mutable term: int, - mutable version: int, -} - -/** Cluster configuration */ -type clusterConfig = { - clusterId: string, - nodeId: string, - bindAddress: string, - bindPort: int, - seedNodes: array, - heartbeatIntervalMs: float, - electionTimeoutMs: float, -} - -/** Default cluster config */ -let defaultConfig: clusterConfig = { - clusterId: "lith-cluster", - nodeId: "", - bindAddress: "0.0.0.0", - bindPort: 7946, - seedNodes: [], - heartbeatIntervalMs: 1000.0, - electionTimeoutMs: 5000.0, -} - -/** Create cluster state */ -let makeState = (~clusterId: string): clusterState => { - { - clusterId, - nodes: [], - leader: None, - term: 0, - version: 0, - } -} - -/** Add node to cluster */ -let addNode = (state: clusterState, node: nodeInfo): unit => { - // Check if node already exists - let exists = state.nodes->Array.some(n => n.id == node.id) - if !exists { - state.nodes->Array.push(node)->ignore - state.version = state.version + 1 - } -} - -/** Remove node from cluster */ -let removeNode = (state: clusterState, nodeId: string): unit => { - state.nodes = state.nodes->Array.filter(n => n.id != nodeId) - state.version = state.version + 1 - - // Clear leader if it was the removed node - switch state.leader { - | Some(id) if id == nodeId => state.leader = None - | _ => () - } -} - -/** Update node status */ -let updateNodeStatus = (state: clusterState, nodeId: string, status: nodeStatus): unit => { - state.nodes->Array.forEach(node => { - if node.id == nodeId { - // Would need mutable node for real implementation - () - } - }) - state.version = state.version + 1 -} - -/** Get active nodes */ -let getActiveNodes = (state: clusterState): array => { - state.nodes->Array.filter(n => n.status == Active) -} - -/** Get node by ID */ -let getNode = (state: clusterState, nodeId: string): option => { - state.nodes->Array.find(n => n.id == nodeId) -} - -/** Check if node is leader */ -let isLeader = (state: clusterState, nodeId: string): bool => { - switch state.leader { - | Some(id) => id == nodeId - | None => false - } -} - -/** Set leader */ -let setLeader = (state: clusterState, nodeId: string, term: int): unit => { - state.leader = Some(nodeId) - state.term = term - state.version = state.version + 1 -} - -/** Generate node ID */ -let generateNodeId = (): string => { - let timestamp = Js.Date.now()->Float.toString - let random = Js.Math.random()->Float.toString - `node-${timestamp}-${random}` -} - -/** Node info to JSON */ -let nodeInfoToJson = (node: nodeInfo): Js.Json.t => { - let statusToString = s => - switch s { - | Starting => "starting" - | Joining => "joining" - | Active => "active" - | Leaving => "leaving" - | Down => "down" - } - - let roleToString = r => - switch r { - | Leader => "leader" - | Follower => "follower" - | Candidate => "candidate" - } - - let obj = Js.Dict.empty() - Js.Dict.set(obj, "id", Js.Json.string(node.id)) - Js.Dict.set(obj, "address", Js.Json.string(node.address)) - Js.Dict.set(obj, "port", Js.Json.number(Int.toFloat(node.port))) - Js.Dict.set(obj, "status", Js.Json.string(statusToString(node.status))) - Js.Dict.set(obj, "role", Js.Json.string(roleToString(node.role))) - Js.Dict.set(obj, "lastHeartbeat", Js.Json.number(node.lastHeartbeat)) - Js.Json.object_(obj) -} - -/** Cluster state to JSON */ -let stateToJson = (state: clusterState): Js.Json.t => { - let obj = Js.Dict.empty() - Js.Dict.set(obj, "clusterId", Js.Json.string(state.clusterId)) - Js.Dict.set(obj, "nodes", Js.Json.array(state.nodes->Array.map(nodeInfoToJson))) - switch state.leader { - | Some(id) => Js.Dict.set(obj, "leader", Js.Json.string(id)) - | None => () - } - Js.Dict.set(obj, "term", Js.Json.number(Int.toFloat(state.term))) - Js.Dict.set(obj, "version", Js.Json.number(Int.toFloat(state.version))) - Js.Json.object_(obj) -} diff --git a/lithoglyph/distributed/src/Lith_Distributed_Consensus.res b/lithoglyph/distributed/src/Lith_Distributed_Consensus.res deleted file mode 100644 index 0d6f05a6..00000000 --- a/lithoglyph/distributed/src/Lith_Distributed_Consensus.res +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Distributed Consensus - * - * Raft-based consensus for distributed Lith clusters - */ - -/** Log entry type */ -type logEntryType = - | Command(Js.Json.t) - | Configuration(array) - | NoOp - -/** Log entry */ -type logEntry = { - index: int, - term: int, - entryType: logEntryType, - timestamp: float, -} - -/** Raft state */ -type raftState = - | Follower - | Candidate - | Leader - -/** Vote request */ -type voteRequest = { - term: int, - candidateId: string, - lastLogIndex: int, - lastLogTerm: int, -} - -/** Vote response */ -type voteResponse = { - term: int, - voteGranted: bool, -} - -/** Append entries request */ -type appendEntriesRequest = { - term: int, - leaderId: string, - prevLogIndex: int, - prevLogTerm: int, - entries: array, - leaderCommit: int, -} - -/** Append entries response */ -type appendEntriesResponse = { - term: int, - success: bool, - matchIndex: int, -} - -/** Consensus node */ -type consensusNode = { - nodeId: string, - mutable state: raftState, - mutable currentTerm: int, - mutable votedFor: option, - mutable log: array, - mutable commitIndex: int, - mutable lastApplied: int, - // Leader state - mutable nextIndex: Js.Dict.t, - mutable matchIndex: Js.Dict.t, - // Timing - mutable lastHeartbeat: float, - electionTimeout: float, - heartbeatInterval: float, -} - -/** Create consensus node */ -let make = ( - ~nodeId: string, - ~electionTimeout: float=5000.0, - ~heartbeatInterval: float=1000.0, -): consensusNode => { - { - nodeId, - state: Follower, - currentTerm: 0, - votedFor: None, - log: [], - commitIndex: 0, - lastApplied: 0, - nextIndex: Js.Dict.empty(), - matchIndex: Js.Dict.empty(), - lastHeartbeat: Js.Date.now(), - electionTimeout, - heartbeatInterval, - } -} - -/** Get last log index */ -let getLastLogIndex = (node: consensusNode): int => { - let len = Array.length(node.log) - if len == 0 { - 0 - } else { - switch node.log->Array.get(len - 1) { - | Some(entry) => entry.index - | None => 0 - } - } -} - -/** Get last log term */ -let getLastLogTerm = (node: consensusNode): int => { - let len = Array.length(node.log) - if len == 0 { - 0 - } else { - switch node.log->Array.get(len - 1) { - | Some(entry) => entry.term - | None => 0 - } - } -} - -/** Handle vote request */ -let handleVoteRequest = (node: consensusNode, request: voteRequest): voteResponse => { - // Update term if request has higher term - if request.term > node.currentTerm { - node.currentTerm = request.term - node.state = Follower - node.votedFor = None - } - - // Check if we can grant vote - let logOk = - request.lastLogTerm > getLastLogTerm(node) || - (request.lastLogTerm == getLastLogTerm(node) && request.lastLogIndex >= getLastLogIndex(node)) - - let canVote = switch node.votedFor { - | None => true - | Some(id) => id == request.candidateId - } - - let voteGranted = request.term >= node.currentTerm && logOk && canVote - - if voteGranted { - node.votedFor = Some(request.candidateId) - node.lastHeartbeat = Js.Date.now() - } - - {term: node.currentTerm, voteGranted} -} - -/** Handle append entries */ -let handleAppendEntries = (node: consensusNode, request: appendEntriesRequest): appendEntriesResponse => { - // Update term if request has higher term - if request.term > node.currentTerm { - node.currentTerm = request.term - node.state = Follower - node.votedFor = None - } - - // Reset heartbeat timer - node.lastHeartbeat = Js.Date.now() - - // Reject if term is stale - if request.term < node.currentTerm { - {term: node.currentTerm, success: false, matchIndex: 0} - } else { - // Check log consistency - let logConsistent = if request.prevLogIndex == 0 { - true - } else { - switch node.log->Array.get(request.prevLogIndex - 1) { - | Some(entry) => entry.term == request.prevLogTerm - | None => false - } - } - - if !logConsistent { - {term: node.currentTerm, success: false, matchIndex: 0} - } else { - // Append entries - request.entries->Array.forEach(entry => { - // Remove conflicting entries and append new ones - node.log = node.log->Array.filter(e => e.index < entry.index) - node.log->Array.push(entry)->ignore - }) - - // Update commit index - if request.leaderCommit > node.commitIndex { - let lastNewEntry = getLastLogIndex(node) - node.commitIndex = min(request.leaderCommit, lastNewEntry) - } - - {term: node.currentTerm, success: true, matchIndex: getLastLogIndex(node)} - } - } -} - -/** Start election */ -let startElection = (node: consensusNode): voteRequest => { - node.currentTerm = node.currentTerm + 1 - node.state = Candidate - node.votedFor = Some(node.nodeId) - node.lastHeartbeat = Js.Date.now() - - { - term: node.currentTerm, - candidateId: node.nodeId, - lastLogIndex: getLastLogIndex(node), - lastLogTerm: getLastLogTerm(node), - } -} - -/** Become leader */ -let becomeLeader = (node: consensusNode, peers: array): unit => { - node.state = Leader - - // Initialize next and match indices for all peers - let lastIndex = getLastLogIndex(node) + 1 - peers->Array.forEach(peer => { - Js.Dict.set(node.nextIndex, peer, lastIndex) - Js.Dict.set(node.matchIndex, peer, 0) - }) -} - -/** Append command to log (leader only) */ -let appendCommand = (node: consensusNode, command: Js.Json.t): option => { - if node.state != Leader { - None - } else { - let entry = { - index: getLastLogIndex(node) + 1, - term: node.currentTerm, - entryType: Command(command), - timestamp: Js.Date.now(), - } - node.log->Array.push(entry)->ignore - Some(entry) - } -} - -/** Check if election timeout elapsed */ -let electionTimeoutElapsed = (node: consensusNode): bool => { - Js.Date.now() -. node.lastHeartbeat > node.electionTimeout -} - -/** Get state string */ -let stateToString = (state: raftState): string => { - switch state { - | Follower => "follower" - | Candidate => "candidate" - | Leader => "leader" - } -} diff --git a/lithoglyph/distributed/src/Lith_Distributed_Replication.res b/lithoglyph/distributed/src/Lith_Distributed_Replication.res deleted file mode 100644 index e3dd461c..00000000 --- a/lithoglyph/distributed/src/Lith_Distributed_Replication.res +++ /dev/null @@ -1,234 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Replication - * - * Data replication and synchronization across nodes - */ - -/** Replication mode */ -type replicationMode = - | Synchronous // Wait for all replicas - | Asynchronous // Fire and forget - | SemiSync // Wait for quorum - -/** Consistency level */ -type consistencyLevel = - | One // One node must respond - | Quorum // Majority must respond - | All // All nodes must respond - | LocalQuorum // Local datacenter quorum - -/** Replication event type */ -type replicationEventType = - | Write - | Update - | Delete - | Snapshot - -/** Replication event */ -type replicationEvent = { - id: string, - eventType: replicationEventType, - collection: string, - documentId: string, - data: option, - timestamp: float, - sourceNode: string, - sequenceNumber: int, -} - -/** Replication configuration */ -type replicationConfig = { - mode: replicationMode, - readConsistency: consistencyLevel, - writeConsistency: consistencyLevel, - replicationFactor: int, - syncIntervalMs: float, - conflictResolution: string, // "last-write-wins", "vector-clock", "custom" -} - -/** Default replication config */ -let defaultConfig: replicationConfig = { - mode: SemiSync, - readConsistency: One, - writeConsistency: Quorum, - replicationFactor: 3, - syncIntervalMs: 100.0, - conflictResolution: "last-write-wins", -} - -/** Replica status */ -type replicaStatus = { - nodeId: string, - lastSequence: int, - lag: int, - isHealthy: bool, - lastSync: float, -} - -/** Replication manager */ -type replicationManager = { - config: replicationConfig, - nodeId: string, - mutable sequence: int, - mutable pendingEvents: array, - mutable replicaStatus: Js.Dict.t, -} - -/** Create replication manager */ -let make = (~config: replicationConfig=defaultConfig, ~nodeId: string): replicationManager => { - { - config, - nodeId, - sequence: 0, - pendingEvents: [], - replicaStatus: Js.Dict.empty(), - } -} - -/** Generate event ID */ -let generateEventId = (): string => { - let timestamp = Js.Date.now()->Float.toString - let random = Js.Math.random()->Float.toString - `evt-${timestamp}-${random}` -} - -/** Create replication event */ -let createEvent = ( - manager: replicationManager, - eventType: replicationEventType, - collection: string, - documentId: string, - data: option, -): replicationEvent => { - manager.sequence = manager.sequence + 1 - - { - id: generateEventId(), - eventType, - collection, - documentId, - data, - timestamp: Js.Date.now(), - sourceNode: manager.nodeId, - sequenceNumber: manager.sequence, - } -} - -/** Queue event for replication */ -let queueEvent = (manager: replicationManager, event: replicationEvent): unit => { - manager.pendingEvents->Array.push(event)->ignore -} - -/** Get pending events */ -let getPendingEvents = (manager: replicationManager): array => { - manager.pendingEvents -} - -/** Clear acknowledged events */ -let acknowledgeEvents = (manager: replicationManager, upToSequence: int): unit => { - manager.pendingEvents = manager.pendingEvents->Array.filter(e => - e.sequenceNumber > upToSequence - ) -} - -/** Update replica status */ -let updateReplicaStatus = ( - manager: replicationManager, - nodeId: string, - lastSequence: int, - isHealthy: bool, -): unit => { - let lag = manager.sequence - lastSequence - let status: replicaStatus = { - nodeId, - lastSequence, - lag, - isHealthy, - lastSync: Js.Date.now(), - } - Js.Dict.set(manager.replicaStatus, nodeId, status) -} - -/** Get replica status */ -let getReplicaStatus = (manager: replicationManager, nodeId: string): option => { - Js.Dict.get(manager.replicaStatus, nodeId) -} - -/** Calculate required responses for consistency level */ -let requiredResponses = (manager: replicationManager, level: consistencyLevel): int => { - let rf = manager.config.replicationFactor - switch level { - | One => 1 - | Quorum => rf / 2 + 1 - | All => rf - | LocalQuorum => rf / 2 + 1 // Simplified - } -} - -/** Check if write is successful */ -let isWriteSuccessful = (manager: replicationManager, ackCount: int): bool => { - let required = requiredResponses(manager, manager.config.writeConsistency) - ackCount >= required -} - -/** Check if read is successful */ -let isReadSuccessful = (manager: replicationManager, responseCount: int): bool => { - let required = requiredResponses(manager, manager.config.readConsistency) - responseCount >= required -} - -/** Get replication lag statistics */ -type lagStats = { - maxLag: int, - avgLag: float, - healthyReplicas: int, - totalReplicas: int, -} - -let getLagStats = (manager: replicationManager): lagStats => { - let statuses = Js.Dict.values(manager.replicaStatus) - let len = Array.length(statuses) - - if len == 0 { - {maxLag: 0, avgLag: 0.0, healthyReplicas: 0, totalReplicas: 0} - } else { - let maxLag = statuses->Array.map(s => s.lag)->Array.reduce(0, (a, b) => max(a, b)) - let totalLag = statuses->Array.map(s => s.lag)->Array.reduce(0, (a, b) => a + b) - let avgLag = Int.toFloat(totalLag) /. Int.toFloat(len) - let healthy = statuses->Array.filter(s => s.isHealthy)->Array.length - - { - maxLag, - avgLag, - healthyReplicas: healthy, - totalReplicas: len, - } - } -} - -/** Event to JSON */ -let eventToJson = (event: replicationEvent): Js.Json.t => { - let typeToString = t => - switch t { - | Write => "write" - | Update => "update" - | Delete => "delete" - | Snapshot => "snapshot" - } - - let obj = Js.Dict.empty() - Js.Dict.set(obj, "id", Js.Json.string(event.id)) - Js.Dict.set(obj, "type", Js.Json.string(typeToString(event.eventType))) - Js.Dict.set(obj, "collection", Js.Json.string(event.collection)) - Js.Dict.set(obj, "documentId", Js.Json.string(event.documentId)) - switch event.data { - | Some(d) => Js.Dict.set(obj, "data", d) - | None => () - } - Js.Dict.set(obj, "timestamp", Js.Json.number(event.timestamp)) - Js.Dict.set(obj, "sourceNode", Js.Json.string(event.sourceNode)) - Js.Dict.set(obj, "sequenceNumber", Js.Json.number(Int.toFloat(event.sequenceNumber))) - Js.Json.object_(obj) -} diff --git a/lithoglyph/distributed/src/Lith_Distributed_Sharding.res b/lithoglyph/distributed/src/Lith_Distributed_Sharding.res deleted file mode 100644 index 4b1151a2..00000000 --- a/lithoglyph/distributed/src/Lith_Distributed_Sharding.res +++ /dev/null @@ -1,236 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -/** - * Lith Sharding - * - * Consistent hashing and shard management for distributed data - */ - -/** Shard status */ -type shardStatus = - | Initializing - | Active - | Migrating - | Inactive - -/** Shard info */ -type shardInfo = { - id: int, - status: shardStatus, - primaryNode: string, - replicaNodes: array, - keyRangeStart: int, - keyRangeEnd: int, - documentCount: int, -} - -/** Sharding strategy */ -type shardingStrategy = - | Hash - | Range - | Directory - -/** Sharding configuration */ -type shardingConfig = { - strategy: shardingStrategy, - numShards: int, - replicationFactor: int, - virtualNodes: int, // For consistent hashing -} - -/** Default sharding config */ -let defaultConfig: shardingConfig = { - strategy: Hash, - numShards: 16, - replicationFactor: 3, - virtualNodes: 150, -} - -/** Hash ring node */ -type hashRingNode = { - hash: int, - nodeId: string, - shardId: int, -} - -/** Shard manager */ -type shardManager = { - config: shardingConfig, - mutable shards: array, - mutable hashRing: array, - mutable nodeToShards: Js.Dict.t>, -} - -/** Simple hash function (FNV-1a inspired) */ -let hashKey = (key: string): int => { - let hash = ref(2166136261) - for i in 0 to String.length(key) - 1 { - let charCode = String.charCodeAt(key, i)->Option.getOr(0.0)->Float.toInt - hash := Int.lxor(hash.contents, charCode) - // Multiply by FNV prime (simplified) - hash := hash.contents * 16777619 - } - Int.abs(hash.contents) -} - -/** Create shard manager */ -let make = (~config: shardingConfig=defaultConfig): shardManager => { - { - config, - shards: [], - hashRing: [], - nodeToShards: Js.Dict.empty(), - } -} - -/** Initialize shards */ -let initializeShards = (manager: shardManager, nodes: array): unit => { - let numNodes = Array.length(nodes) - if numNodes == 0 { - () - } else { - // Create shards - let shardsPerRange = 2147483647 / manager.config.numShards - - for i in 0 to manager.config.numShards - 1 { - let primaryIdx = mod(i, numNodes) - let primaryNode = nodes->Array.getUnsafe(primaryIdx) - - // Assign replica nodes - let replicaNodes: array = [] - for j in 1 to min(manager.config.replicationFactor - 1, numNodes - 1) { - let replicaIdx = mod(primaryIdx + j, numNodes) - replicaNodes->Array.push(nodes->Array.getUnsafe(replicaIdx))->ignore - } - - let shard: shardInfo = { - id: i, - status: Active, - primaryNode, - replicaNodes, - keyRangeStart: i * shardsPerRange, - keyRangeEnd: (i + 1) * shardsPerRange - 1, - documentCount: 0, - } - manager.shards->Array.push(shard)->ignore - } - - // Build hash ring with virtual nodes - nodes->Array.forEachWithIndex((node, nodeIdx) => { - for v in 0 to manager.config.virtualNodes - 1 { - let virtualKey = `${node}:${Int.toString(v)}` - let hash = hashKey(virtualKey) - let shardId = mod(nodeIdx, manager.config.numShards) - - manager.hashRing->Array.push({hash, nodeId: node, shardId})->ignore - } - }) - - // Sort hash ring - manager.hashRing->Array.sort((a, b) => a.hash - b.hash) - - // Build node to shards mapping - manager.shards->Array.forEach(shard => { - // Primary - switch Js.Dict.get(manager.nodeToShards, shard.primaryNode) { - | Some(shards) => shards->Array.push(shard.id)->ignore - | None => Js.Dict.set(manager.nodeToShards, shard.primaryNode, [shard.id]) - } - - // Replicas - shard.replicaNodes->Array.forEach(node => { - switch Js.Dict.get(manager.nodeToShards, node) { - | Some(shards) => shards->Array.push(shard.id)->ignore - | None => Js.Dict.set(manager.nodeToShards, node, [shard.id]) - } - }) - }) - } -} - -/** Get shard for key (hash strategy) */ -let getShardForKey = (manager: shardManager, key: string): option => { - switch manager.config.strategy { - | Hash => { - let hash = hashKey(key) - let shardId = mod(hash, manager.config.numShards) - manager.shards->Array.get(shardId) - } - | Range => { - // Range-based: use first character - let firstChar = String.charCodeAt(key, 0)->Option.getOr(0.0)->Float.toInt - let shardId = mod(firstChar, manager.config.numShards) - manager.shards->Array.get(shardId) - } - | Directory => { - // Directory-based: would look up in directory - // For now, fall back to hash - let hash = hashKey(key) - let shardId = mod(hash, manager.config.numShards) - manager.shards->Array.get(shardId) - } - } -} - -/** Get node for key using consistent hashing */ -let getNodeForKey = (manager: shardManager, key: string): option => { - let ringLen = Array.length(manager.hashRing) - if ringLen == 0 { - None - } else { - let hash = hashKey(key) - - // Binary search for first node with hash >= key hash - let rec binarySearch = (low: int, high: int): int => { - if low >= high { - low - } else { - let mid = (low + high) / 2 - switch manager.hashRing->Array.get(mid) { - | Some(node) => - if node.hash < hash { - binarySearch(mid + 1, high) - } else { - binarySearch(low, mid) - } - | None => low - } - } - } - - let idx = binarySearch(0, ringLen) - let finalIdx = if idx >= ringLen { - 0 - } else { - idx - } - - manager.hashRing->Array.get(finalIdx)->Option.map(n => n.nodeId) - } -} - -/** Get shards for node */ -let getShardsForNode = (manager: shardManager, nodeId: string): array => { - Js.Dict.get(manager.nodeToShards, nodeId)->Option.getOr([]) -} - -/** Get shard statistics */ -type shardStats = { - totalShards: int, - activeShards: int, - migratingShards: int, - totalDocuments: int, -} - -let getStats = (manager: shardManager): shardStats => { - let active = manager.shards->Array.filter(s => s.status == Active)->Array.length - let migrating = manager.shards->Array.filter(s => s.status == Migrating)->Array.length - let docs = manager.shards->Array.map(s => s.documentCount)->Array.reduce(0, (a, b) => a + b) - - { - totalShards: Array.length(manager.shards), - activeShards: active, - migratingShards: migrating, - totalDocuments: docs, - } -} diff --git a/lithoglyph/docs/API-REFERENCE.adoc b/lithoglyph/docs/API-REFERENCE.adoc deleted file mode 100644 index 994208f7..00000000 --- a/lithoglyph/docs/API-REFERENCE.adoc +++ /dev/null @@ -1,1809 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith API Reference -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font -:sectnums: - -[.lead] -Complete API reference for Lith's programmatic interfaces: GQL query language, Zig ABI, HTTP REST API, and client libraries. - -toc::[] - -== Overview - -Lith exposes multiple interface layers for different use cases: - -[cols="1,2,1,1"] -|=== -| Layer | Description | Status | Use Case - -| **GQL** -| High-level narrative query language -| Specified -| Interactive queries, scripts - -| **Zig ABI** -| Form.Bridge FFI for language bindings -| Specified -| Embedded use, language bindings - -| **HTTP/REST** -| Network API with JSON -| Planned 🚧 -| Web applications, microservices - -| **gRPC** -| High-performance binary RPC -| Planned 🚧 -| Low-latency services -|=== - -== GQL API - -The Lith Query Language (GQL) is the primary interface for interacting with Lith. See link:../spec/gql.adoc[GQL Specification] for the complete grammar. - -=== Connection - -[source,bash] ----- -# Interactive shell -lith shell mydb/ - -# Execute query from file -lith query mydb/ -f query.gql - -# Execute inline query -lith query mydb/ -e "SELECT * FROM evidence LIMIT 10" - -# Connect to remote server -lith shell --host lith.example.com --port 5432 --tls ----- - -=== Collection Operations - -==== CREATE COLLECTION - -Create a new document or edge collection. - -[source,gql] ----- --- Document collection with schema -CREATE COLLECTION evidence ( - title STRING NOT NULL, - source STRING, - content STRING, - score PROMPT_SCORE, - metadata JSON, - created_at TIMESTAMP DEFAULT NOW() -) -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Initialize evidence storage for case #2024-001" -}; - --- Edge collection (connects documents) -CREATE EDGE COLLECTION cites ( - citation_type STRING, - page_number INTEGER, - confidence PROMPT_SCORE -) -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Track citation relationships" -}; - --- Collection with constraints -CREATE COLLECTION users ( - email STRING NOT NULL UNIQUE, - name STRING NOT NULL, - role STRING DEFAULT 'viewer', - active BOOLEAN DEFAULT TRUE -) -WITH CONSTRAINTS { - role IN ('viewer', 'editor', 'admin'), - email MATCHES '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$' -} -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "User management for access control" -}; ----- - -**Response:** -[source,json] ----- -{ - "status": "success", - "collection": "evidence", - "operation": "CREATE_COLLECTION", - "journal_sequence": 1001 -} ----- - -==== DROP COLLECTION - -Remove a collection and all its documents. - -[source,gql] ----- --- Drop with provenance (soft delete, recoverable) -DROP COLLECTION old_evidence -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Migrated to new schema, old data archived" -}; - --- Hard drop (irreversible, requires confirmation) -DROP COLLECTION temp_import HARD -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Temporary import data no longer needed" -} -CONFIRM IRREVERSIBLE; ----- - -==== ALTER COLLECTION - -Modify collection schema or settings. - -[source,gql] ----- --- Add a new field -ALTER COLLECTION evidence ADD COLUMN ( - verified BOOLEAN DEFAULT FALSE -) -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Add verification tracking per policy update" -}; - --- Add constraint -ALTER COLLECTION evidence ADD CONSTRAINT ( - score >= 0 AND score <= 100 -) -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Enforce PROMPT_SCORE range" -}; - --- Rename collection -ALTER COLLECTION evidence RENAME TO case_evidence -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Disambiguate from other evidence types" -}; ----- - -=== Document Operations - -==== INSERT - -Create new documents with mandatory provenance. - -[source,gql] ----- --- Single document -INSERT INTO evidence { - title: "Financial Report Q4 2024", - source: "SEC Filing", - content: "...", - score: 85, - metadata: {"filing_id": "0001234567-24-000123"} -} -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Primary source document for investigation #INV-2024-042" -}; - --- Multiple documents -INSERT INTO evidence [ - {title: "Document A", source: "Archive", score: 70}, - {title: "Document B", source: "Archive", score: 65}, - {title: "Document C", source: "Archive", score: 80} -] -WITH PROVENANCE { - actor: "service:bulk-ingestion", - rationale: "Batch import from national archives request #NA-2024-100" -}; - --- Insert with generated ID returned -INSERT INTO evidence { - title: "New Evidence" -} -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Initial entry" -} -RETURNING _id, _created_at; ----- - -**Response:** -[source,json] ----- -{ - "status": "success", - "operation": "INSERT", - "collection": "evidence", - "documents_inserted": 1, - "ids": ["doc_abc123xyz"], - "journal_sequence": 1002 -} ----- - -==== SELECT - -Query documents with filtering, projection, and ordering. - -[source,gql] ----- --- Basic query -SELECT * FROM evidence; - --- With projection (specific fields) -SELECT title, source, score FROM evidence; - --- With filtering -SELECT * FROM evidence -WHERE score >= 70 AND source = "SEC Filing"; - --- With ordering and limit -SELECT title, score FROM evidence -WHERE verified = TRUE -ORDER BY score DESC -LIMIT 10; - --- With offset for pagination -SELECT * FROM evidence -ORDER BY _created_at DESC -LIMIT 20 OFFSET 40; - --- JSON field access -SELECT title, metadata.filing_id FROM evidence -WHERE metadata.category = "financial"; - --- Pattern matching -SELECT * FROM evidence -WHERE title LIKE "%Report%" - AND source IN ("SEC Filing", "Court Record", "FOIA Response"); - --- Date filtering -SELECT * FROM evidence -WHERE _created_at >= "2024-01-01" - AND _created_at < "2025-01-01"; - --- Null handling -SELECT * FROM evidence -WHERE verified IS NULL - OR verified = FALSE; ----- - -**Response:** -[source,json] ----- -{ - "status": "success", - "operation": "SELECT", - "collection": "evidence", - "count": 42, - "results": [ - { - "_id": "doc_abc123", - "title": "Financial Report Q4 2024", - "source": "SEC Filing", - "score": 85, - "_created_at": "2024-06-15T14:30:00Z" - } - ] -} ----- - -==== UPDATE - -Modify existing documents. - -[source,gql] ----- --- Update single document by ID -UPDATE evidence -SET verified = TRUE, - score = 90 -WHERE _id = "doc_abc123" -WITH PROVENANCE { - actor: "user:reviewer@example.com", - rationale: "Verified against primary source, upgraded confidence" -}; - --- Update multiple documents -UPDATE evidence -SET verified = TRUE -WHERE source = "Court Record" AND verified IS NULL -WITH PROVENANCE { - actor: "user:legal@example.com", - rationale: "Batch verification of court records" -}; - --- Increment/modify -UPDATE evidence -SET score = score + 5, - metadata = JSON_SET(metadata, '$.review_count', - COALESCE(metadata.review_count, 0) + 1) -WHERE _id = "doc_abc123" -WITH PROVENANCE { - actor: "user:reviewer@example.com", - rationale: "Additional corroboration found" -}; - --- Update with RETURNING -UPDATE evidence -SET score = 95 -WHERE _id = "doc_abc123" -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Final score after review" -} -RETURNING _id, score, _updated_at; ----- - -**Response:** -[source,json] ----- -{ - "status": "success", - "operation": "UPDATE", - "collection": "evidence", - "documents_updated": 1, - "journal_sequence": 1003 -} ----- - -==== DELETE - -Remove documents (soft delete by default). - -[source,gql] ----- --- Soft delete (recoverable) -DELETE FROM evidence -WHERE _id = "doc_abc123" -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Duplicate entry, superseded by doc_def456" -}; - --- Bulk soft delete -DELETE FROM evidence -WHERE verified = FALSE AND _created_at < "2023-01-01" -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Cleanup unverified legacy data per retention policy" -}; - --- Hard delete (irreversible) -DELETE FROM evidence -WHERE _id = "doc_sensitive" -HARD -WITH PROVENANCE { - actor: "user:dpo@example.com", - rationale: "GDPR erasure request #ER-2024-042" -} -CONFIRM IRREVERSIBLE; ----- - -=== Edge Operations - -==== CREATE EDGE - -Connect documents with typed relationships. - -[source,gql] ----- --- Create edge between documents -CREATE EDGE cites FROM "doc_abc123" TO "doc_def456" { - citation_type: "direct_quote", - page_number: 42, - confidence: 95 -} -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Document A directly quotes Document B on page 42" -}; - --- Create edge from query results -CREATE EDGE references -FROM (SELECT _id FROM evidence WHERE source = "SEC Filing") -TO "doc_regulatory_framework" -{ - reference_type: "regulatory_basis" -} -WITH PROVENANCE { - actor: "service:auto-linker", - rationale: "Auto-detected regulatory references" -}; ----- - -==== TRAVERSE - -Navigate graph relationships. - -[source,gql] ----- --- Outbound traversal (what does this document cite?) -SELECT * FROM evidence -WHERE _id = "doc_abc123" -TRAVERSE cites OUTBOUND DEPTH 1; - --- Inbound traversal (what cites this document?) -SELECT * FROM evidence -WHERE _id = "doc_abc123" -TRAVERSE cites INBOUND DEPTH 1; - --- Deep traversal (citation chain) -SELECT * FROM evidence -WHERE _id = "doc_abc123" -TRAVERSE cites OUTBOUND DEPTH 5 -WITH PATH; -- Include traversal path in results - --- Multi-edge traversal -SELECT * FROM evidence -WHERE _id = "doc_abc123" -TRAVERSE (cites, references, contradicts) OUTBOUND DEPTH 3; - --- Filtered traversal -SELECT * FROM evidence -WHERE _id = "doc_abc123" -TRAVERSE cites OUTBOUND DEPTH 3 -WHERE edge.confidence >= 80 AND target.verified = TRUE; ----- - -**Response with PATH:** -[source,json] ----- -{ - "status": "success", - "results": [ - { - "_id": "doc_xyz789", - "title": "Original Source", - "_path": [ - {"from": "doc_abc123", "edge": "cites", "to": "doc_def456"}, - {"from": "doc_def456", "edge": "cites", "to": "doc_xyz789"} - ], - "_depth": 2 - } - ] -} ----- - -=== Introspection Operations - -==== INTROSPECT SCHEMA - -Examine collection schemas. - -[source,gql] ----- --- All collections -INTROSPECT SCHEMA; - --- Specific collection -INTROSPECT SCHEMA evidence; - --- With history -INTROSPECT SCHEMA evidence WITH HISTORY; ----- - -**Response:** -[source,json] ----- -{ - "collection": "evidence", - "type": "document", - "schema_version": 3, - "fields": [ - {"name": "title", "type": "STRING", "nullable": false}, - {"name": "source", "type": "STRING", "nullable": true}, - {"name": "score", "type": "PROMPT_SCORE", "nullable": true}, - {"name": "verified", "type": "BOOLEAN", "default": false} - ], - "constraints": [ - {"type": "RANGE", "field": "score", "min": 0, "max": 100} - ], - "created_at": "2024-01-15T10:00:00Z", - "updated_at": "2024-06-15T14:30:00Z" -} ----- - -==== INTROSPECT CONSTRAINTS - -View active constraints. - -[source,gql] ----- --- All constraints -INTROSPECT CONSTRAINTS; - --- Specific collection -INTROSPECT CONSTRAINTS evidence; ----- - -**Response:** -[source,json] ----- -{ - "collection": "evidence", - "constraints": [ - { - "id": "con_001", - "type": "RANGE", - "field": "score", - "expression": "score >= 0 AND score <= 100", - "enforced": true, - "rationale": "PROMPT_SCORE must be 0-100" - }, - { - "id": "con_002", - "type": "NOT_NULL", - "field": "title", - "enforced": true - } - ] -} ----- - -==== INTROSPECT JOURNAL - -Query the operation journal. - -[source,gql] ----- --- Recent entries -INTROSPECT JOURNAL LIMIT 100; - --- Since specific sequence -INTROSPECT JOURNAL SINCE 1000 LIMIT 50; - --- For specific collection -INTROSPECT JOURNAL -WHERE collection = "evidence" -LIMIT 50; - --- For specific actor -INTROSPECT JOURNAL -WHERE actor = "user:analyst@example.com" -LIMIT 50; - --- Time range -INTROSPECT JOURNAL -WHERE timestamp >= "2024-06-01" AND timestamp < "2024-07-01"; ----- - -**Response:** -[source,json] ----- -{ - "entries": [ - { - "sequence": 1003, - "timestamp": "2024-06-15T14:35:00Z", - "operation": "UPDATE", - "collection": "evidence", - "document_id": "doc_abc123", - "changes": { - "score": {"old": 85, "new": 90}, - "verified": {"old": null, "new": true} - }, - "provenance": { - "actor": "user:reviewer@example.com", - "rationale": "Verified against primary source" - }, - "inverse": { - "operation": "UPDATE", - "changes": {"score": 85, "verified": null} - } - } - ] -} ----- - -==== INTROSPECT HISTORY - -View document change history. - -[source,gql] ----- --- Full history of a document -INTROSPECT HISTORY "doc_abc123"; - --- History with diffs -INTROSPECT HISTORY "doc_abc123" WITH DIFFS; - --- History for time range -INTROSPECT HISTORY "doc_abc123" -WHERE timestamp >= "2024-01-01"; ----- - -==== EXPLAIN - -Show query execution plan. - -[source,gql] ----- -EXPLAIN SELECT * FROM evidence -WHERE score >= 70 AND source = "SEC Filing" -ORDER BY score DESC -LIMIT 10; ----- - -**Response:** -[source,json] ----- -{ - "plan": { - "type": "LIMIT", - "count": 10, - "child": { - "type": "SORT", - "key": "score", - "direction": "DESC", - "child": { - "type": "FILTER", - "conditions": [ - {"field": "score", "op": ">=", "value": 70}, - {"field": "source", "op": "=", "value": "SEC Filing"} - ], - "child": { - "type": "COLLECTION_SCAN", - "collection": "evidence", - "estimated_rows": 1000 - } - } - } - }, - "estimated_cost": 150, - "indexes_used": [] -} ----- - -=== Transaction Operations 🚧 - -[source,gql] ----- --- Begin transaction -BEGIN TRANSACTION; - --- Operations within transaction -INSERT INTO evidence {...} WITH PROVENANCE {...}; -UPDATE evidence SET ... WHERE ... WITH PROVENANCE {...}; -CREATE EDGE cites ... WITH PROVENANCE {...}; - --- Commit -COMMIT -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Atomic update of evidence and citations" -}; - --- Or rollback -ROLLBACK -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Discovered error in import data" -}; ----- - -=== Undo Operations - -[source,gql] ----- --- Undo last operation -UNDO LAST -WITH PROVENANCE { - actor: "user:analyst@example.com", - rationale: "Accidental modification" -}; - --- Undo specific journal entry -UNDO SEQUENCE 1003 -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Reverting unauthorized change" -}; - --- Undo with preview -UNDO SEQUENCE 1003 PREVIEW; ----- - -=== Data Types - -[cols="1,2,2"] -|=== -| Type | Description | Example - -| `STRING` -| Variable-length text -| `"Hello, World"` - -| `INTEGER` -| 64-bit signed integer -| `42`, `-100` - -| `FLOAT` -| 64-bit floating point -| `3.14159`, `-0.001` - -| `BOOLEAN` -| True/false -| `TRUE`, `FALSE` - -| `TIMESTAMP` -| ISO 8601 datetime -| `"2024-06-15T14:30:00Z"` - -| `JSON` -| Arbitrary JSON structure -| `{"key": "value", "nested": {"a": 1}}` - -| `PROMPT_SCORE` -| 0-100 integer for AI confidence -| `85` - -| `ENCRYPTED` -| Encrypted at rest -| (displayed as `[ENCRYPTED]`) -|=== - -=== Operators - -==== Comparison Operators - -[cols="1,2,2"] -|=== -| Operator | Description | Example - -| `=` -| Equal -| `score = 85` - -| `!=`, `<>` -| Not equal -| `status != "deleted"` - -| `<` -| Less than -| `score < 50` - -| `<=` -| Less than or equal -| `score <= 100` - -| `>` -| Greater than -| `score > 70` - -| `>=` -| Greater than or equal -| `created_at >= "2024-01-01"` - -| `IN` -| In set -| `source IN ("A", "B", "C")` - -| `NOT IN` -| Not in set -| `status NOT IN ("deleted", "archived")` - -| `BETWEEN` -| Range (inclusive) -| `score BETWEEN 60 AND 80` - -| `LIKE` -| Pattern match -| `title LIKE "%Report%"` - -| `IS NULL` -| Null check -| `verified IS NULL` - -| `IS NOT NULL` -| Not null check -| `score IS NOT NULL` -|=== - -==== Logical Operators - -[cols="1,2,2"] -|=== -| Operator | Description | Example - -| `AND` -| Logical AND -| `score > 70 AND verified = TRUE` - -| `OR` -| Logical OR -| `source = "A" OR source = "B"` - -| `NOT` -| Logical NOT -| `NOT verified` -|=== - -==== JSON Operators - -[cols="1,2,2"] -|=== -| Operator | Description | Example - -| `.` -| Field access -| `metadata.category` - -| `[]` -| Array access -| `tags[0]` - -| `JSON_SET` -| Set JSON field -| `JSON_SET(metadata, '$.count', 1)` - -| `JSON_EXTRACT` -| Extract JSON value -| `JSON_EXTRACT(metadata, '$.id')` -|=== - -== Zig ABI (Form.Bridge) - -The Form.Bridge provides a stable C ABI for embedding Lith in other applications and creating language bindings. - -=== Type Definitions - -[source,zig] ----- -/// Opaque database handle -pub const LithDb = opaque {}; - -/// Opaque transaction handle -pub const LithTxn = opaque {}; - -/// Opaque query result handle -pub const LithResult = opaque {}; - -/// Status code -pub const LithStatus = enum(i32) { - ok = 0, - - // Client errors (1xxx) - err_not_found = 1001, - err_already_exists = 1002, - err_constraint_violation = 1003, - err_parse_error = 1004, - err_invalid_argument = 1005, - err_type_mismatch = 1006, - err_missing_provenance = 1007, - err_permission_denied = 1008, - - // Server errors (2xxx) - err_internal = 2001, - err_io = 2002, - err_journal_corrupted = 2003, - err_out_of_memory = 2004, - err_timeout = 2005, - - // Transaction errors (3xxx) - err_txn_conflict = 3001, - err_txn_aborted = 3002, - err_txn_not_active = 3003, -}; - -/// Provenance structure -pub const LithProvenance = extern struct { - actor: [*:0]const u8, - actor_len: usize, - rationale: [*:0]const u8, - rationale_len: usize, - timestamp: i64, // Unix timestamp, 0 = auto -}; - -/// Query result metadata -pub const LithResultMeta = extern struct { - status: LithStatus, - count: usize, - journal_sequence: u64, - error_message: [*:0]const u8, - error_message_len: usize, -}; ----- - -=== Database Lifecycle - -[source,zig] ----- -/// Open or create a database -/// Returns: LithStatus (check before using db_out) -pub extern fn lith_open( - path: [*:0]const u8, - path_len: usize, - options: *const LithOpenOptions, - db_out: **LithDb, -) callconv(.C) LithStatus; - -/// Open options -pub const LithOpenOptions = extern struct { - create_if_missing: bool = true, - read_only: bool = false, - journal_sync: bool = true, // fsync after each write - cache_size_mb: u32 = 64, -}; - -/// Close database and free resources -pub extern fn lith_close(db: *LithDb) callconv(.C) LithStatus; - -/// Get database statistics -pub extern fn lith_stats( - db: *LithDb, - stats_out: *LithStats, -) callconv(.C) LithStatus; - -pub const LithStats = extern struct { - collections: u32, - documents: u64, - edges: u64, - journal_entries: u64, - storage_bytes: u64, - cache_hit_ratio: f32, -}; ----- - -=== Query Execution - -[source,zig] ----- -/// Execute GQL query -pub extern fn lith_query( - db: *LithDb, - gql: [*]const u8, - gql_len: usize, - provenance: *const LithProvenance, - result_out: **LithResult, -) callconv(.C) LithStatus; - -/// Get result metadata -pub extern fn lith_result_meta( - result: *LithResult, - meta_out: *LithResultMeta, -) callconv(.C) LithStatus; - -/// Get result row as JSON -pub extern fn lith_result_row( - result: *LithResult, - index: usize, - json_out: *[*]const u8, - json_len_out: *usize, -) callconv(.C) LithStatus; - -/// Iterate result rows -pub extern fn lith_result_next( - result: *LithResult, - json_out: *[*]const u8, - json_len_out: *usize, -) callconv(.C) LithStatus; // Returns err_not_found when exhausted - -/// Free result -pub extern fn lith_result_free(result: *LithResult) callconv(.C) void; ----- - -=== Document Operations - -[source,zig] ----- -/// Insert document (JSON format) -pub extern fn lith_insert( - db: *LithDb, - collection: [*:0]const u8, - collection_len: usize, - doc_json: [*]const u8, - doc_json_len: usize, - provenance: *const LithProvenance, - id_out: *[*]const u8, - id_len_out: *usize, -) callconv(.C) LithStatus; - -/// Get document by ID -pub extern fn lith_get( - db: *LithDb, - collection: [*:0]const u8, - collection_len: usize, - id: [*:0]const u8, - id_len: usize, - doc_out: *[*]const u8, - doc_len_out: *usize, -) callconv(.C) LithStatus; - -/// Update document -pub extern fn lith_update( - db: *LithDb, - collection: [*:0]const u8, - collection_len: usize, - id: [*:0]const u8, - id_len: usize, - updates_json: [*]const u8, - updates_json_len: usize, - provenance: *const LithProvenance, -) callconv(.C) LithStatus; - -/// Delete document -pub extern fn lith_delete( - db: *LithDb, - collection: [*:0]const u8, - collection_len: usize, - id: [*:0]const u8, - id_len: usize, - hard: bool, - provenance: *const LithProvenance, -) callconv(.C) LithStatus; ----- - -=== Transaction Operations - -[source,zig] ----- -/// Begin transaction -pub extern fn lith_txn_begin( - db: *LithDb, - txn_out: **LithTxn, -) callconv(.C) LithStatus; - -/// Execute query within transaction -pub extern fn lith_txn_query( - txn: *LithTxn, - gql: [*]const u8, - gql_len: usize, - provenance: *const LithProvenance, - result_out: **LithResult, -) callconv(.C) LithStatus; - -/// Commit transaction -pub extern fn lith_txn_commit( - txn: *LithTxn, - provenance: *const LithProvenance, -) callconv(.C) LithStatus; - -/// Rollback transaction -pub extern fn lith_txn_rollback( - txn: *LithTxn, - provenance: *const LithProvenance, -) callconv(.C) LithStatus; ----- - -=== Journal Operations - -[source,zig] ----- -/// Get current journal sequence -pub extern fn lith_journal_sequence( - db: *LithDb, - seq_out: *u64, -) callconv(.C) LithStatus; - -/// Read journal entries -pub extern fn lith_journal_read( - db: *LithDb, - since_sequence: u64, - limit: usize, - entries_json_out: *[*]const u8, - entries_json_len_out: *usize, -) callconv(.C) LithStatus; - -/// Undo journal entry -pub extern fn lith_journal_undo( - db: *LithDb, - sequence: u64, - provenance: *const LithProvenance, -) callconv(.C) LithStatus; ----- - -=== Error Handling - -[source,zig] ----- -/// Get detailed error message for last error -pub extern fn lith_error_message( - db: *LithDb, - message_out: *[*]const u8, - message_len_out: *usize, -) callconv(.C) void; - -/// Get error suggestions -pub extern fn lith_error_suggestions( - db: *LithDb, - suggestions_json_out: *[*]const u8, - suggestions_json_len_out: *usize, -) callconv(.C) void; ----- - -=== Memory Management - -[source,zig] ----- -/// Free string allocated by Lith -pub extern fn lith_free_string(ptr: [*]const u8, len: usize) callconv(.C) void; - -/// Free JSON allocated by Lith -pub extern fn lith_free_json(ptr: [*]const u8, len: usize) callconv(.C) void; ----- - -=== Example Usage - -[source,zig] ----- -const std = @import("std"); -const lith = @cImport(@cInclude("lith.h")); - -pub fn main() !void { - // Open database - var db: *lith.LithDb = undefined; - const options = lith.LithOpenOptions{}; - - var status = lith.lith_open( - "mydb/", 5, - &options, - &db - ); - if (status != .ok) { - std.debug.print("Failed to open database\n", .{}); - return; - } - defer _ = lith.lith_close(db); - - // Insert document - const doc = - \\{"title": "Test Document", "score": 85} - ; - const prov = lith.LithProvenance{ - .actor = "user:test@example.com", - .actor_len = 22, - .rationale = "Test insert", - .rationale_len = 11, - .timestamp = 0, // Auto - }; - - var id_ptr: [*]const u8 = undefined; - var id_len: usize = undefined; - - status = lith.lith_insert( - db, - "evidence", 8, - doc.ptr, doc.len, - &prov, - &id_ptr, &id_len, - ); - - if (status == .ok) { - const id = id_ptr[0..id_len]; - std.debug.print("Inserted document: {s}\n", .{id}); - lith.lith_free_string(id_ptr, id_len); - } -} ----- - -== HTTP REST API 🚧 - -The HTTP API provides network access to Lith with JSON request/response format. - -=== Base URL - -[source,text] ----- -https://lith.example.com/v1 ----- - -=== Authentication - -[source,bash] ----- -# API Key (header) -curl -H "X-Lith-API-Key: lith_sk_live_abc123" \ - https://lith.example.com/v1/query - -# JWT Bearer token -curl -H "Authorization: Bearer eyJhbG..." \ - https://lith.example.com/v1/query ----- - -=== Endpoints - -==== POST /v1/query - -Execute GQL query. - -**Request:** -[source,json] ----- -{ - "gql": "SELECT * FROM evidence WHERE score >= 70 LIMIT 10", - "provenance": { - "actor": "user:api-client@example.com", - "rationale": "Dashboard refresh" - }, - "options": { - "timeout_ms": 30000, - "max_results": 1000 - } -} ----- - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "operation": "SELECT", - "collection": "evidence", - "count": 42, - "results": [ - {"_id": "doc_abc123", "title": "...", "score": 85} - ] - }, - "meta": { - "journal_sequence": 1234, - "duration_ms": 45, - "request_id": "req_xyz789" - } -} ----- - -==== GET /v1/collections - -List all collections. - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "collections": [ - { - "name": "evidence", - "type": "document", - "document_count": 1500, - "schema_version": 3 - }, - { - "name": "cites", - "type": "edge", - "edge_count": 3200 - } - ] - } -} ----- - -==== GET /v1/collections/{name} - -Get collection details and schema. - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "name": "evidence", - "type": "document", - "schema": { - "version": 3, - "fields": [ - {"name": "title", "type": "STRING", "nullable": false}, - {"name": "score", "type": "PROMPT_SCORE", "nullable": true} - ] - }, - "constraints": [...], - "indexes": [...], - "stats": { - "document_count": 1500, - "storage_bytes": 15728640 - } - } -} ----- - -==== GET /v1/collections/{name}/documents/{id} - -Get single document. - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "_id": "doc_abc123", - "_created_at": "2024-06-15T14:30:00Z", - "_updated_at": "2024-06-15T15:00:00Z", - "title": "Financial Report Q4 2024", - "score": 85 - } -} ----- - -==== GET /v1/collections/{name}/documents/{id}/history - -Get document history. - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "document_id": "doc_abc123", - "history": [ - { - "sequence": 1003, - "timestamp": "2024-06-15T15:00:00Z", - "operation": "UPDATE", - "changes": {"score": {"old": 80, "new": 85}}, - "provenance": { - "actor": "user:reviewer@example.com", - "rationale": "Score adjustment after review" - } - } - ] - } -} ----- - -==== GET /v1/journal - -Read journal entries. - -**Query Parameters:** -- `since` - Start sequence (default: 0) -- `limit` - Max entries (default: 100, max: 1000) -- `collection` - Filter by collection -- `actor` - Filter by actor - -**Response:** -[source,json] ----- -{ - "status": "success", - "data": { - "entries": [...], - "next_sequence": 1100, - "has_more": true - } -} ----- - -==== GET /v1/health - -Health check endpoint. - -**Response:** -[source,json] ----- -{ - "status": "healthy", - "version": "0.1.0", - "uptime_seconds": 86400, - "checks": { - "storage": "ok", - "journal": "ok", - "cache": "ok" - } -} ----- - -==== GET /v1/health/ready - -Readiness check (for Kubernetes). - -**Response (ready):** -[source,json] ----- -{"ready": true} ----- - -**Response (not ready):** -[source,json] ----- -{ - "ready": false, - "reason": "Journal replay in progress", - "progress": 0.75 -} ----- - -=== Error Responses - -All errors follow a consistent format: - -[source,json] ----- -{ - "status": "error", - "error": { - "code": 1003, - "type": "CONSTRAINT_VIOLATION", - "message": "Value 150 exceeds maximum of 100 for field 'score'", - "field": "score", - "constraint": "PROMPT_SCORE range", - "rationale": "PROMPT_SCORE values must be between 0 and 100 to represent confidence percentages" - }, - "suggestions": [ - "Use a value between 0 and 100", - "If this represents a different scale, consider using INTEGER type" - ], - "meta": { - "request_id": "req_xyz789" - } -} ----- - -=== Rate Limiting - -Rate limit headers are included in all responses: - -[source,text] ----- -X-RateLimit-Limit: 1000 -X-RateLimit-Remaining: 995 -X-RateLimit-Reset: 1718470800 ----- - -When rate limited: - -[source,json] ----- -{ - "status": "error", - "error": { - "code": 429, - "type": "RATE_LIMITED", - "message": "Rate limit exceeded", - "retry_after_seconds": 60 - } -} ----- - -=== Pagination - -Paginated endpoints use cursor-based pagination: - -[source,json] ----- -{ - "data": { - "results": [...], - "pagination": { - "cursor": "eyJzZXEiOjEwMDB9", - "has_more": true, - "total": 5000 - } - } -} ----- - -Use the cursor in subsequent requests: - -[source,bash] ----- -curl "https://lith.example.com/v1/query?cursor=eyJzZXEiOjEwMDB9" ----- - -== gRPC API 🚧 - -High-performance binary RPC for low-latency applications. - -=== Service Definition - -[source,protobuf] ----- -syntax = "proto3"; -package lith.v1; - -service Lith { - // Query operations - rpc Query(QueryRequest) returns (QueryResponse); - rpc StreamQuery(QueryRequest) returns (stream QueryRow); - - // Document operations - rpc Insert(InsertRequest) returns (InsertResponse); - rpc Get(GetRequest) returns (GetResponse); - rpc Update(UpdateRequest) returns (UpdateResponse); - rpc Delete(DeleteRequest) returns (DeleteResponse); - - // Journal operations - rpc StreamJournal(JournalRequest) returns (stream JournalEntry); - - // Health - rpc Health(HealthRequest) returns (HealthResponse); -} - -message Provenance { - string actor = 1; - string rationale = 2; - int64 timestamp = 3; // 0 = auto -} - -message QueryRequest { - string gql = 1; - Provenance provenance = 2; - QueryOptions options = 3; -} - -message QueryOptions { - int32 timeout_ms = 1; - int32 max_results = 2; -} - -message QueryResponse { - Status status = 1; - repeated bytes rows = 2; // JSON-encoded rows - QueryMeta meta = 3; -} - -message QueryMeta { - uint64 journal_sequence = 1; - int32 duration_ms = 2; - int32 row_count = 3; -} ----- - -== Client Libraries 🚧 - -Official client libraries are planned for: - -[cols="1,2,1"] -|=== -| Language | Package | Status - -| **Zig** -| Native (Form.Bridge) -| Available - -| **Rust** -| `lith-rs` -| Planned - -| **Python** -| `lith` -| Planned - -| **JavaScript/TypeScript** -| `@lith/client` -| Planned - -| **Go** -| `github.com/hyperpolymath/lith-go` -| Planned - -| **Java** -| `com.lith:lith-client` -| Planned -|=== - -=== Rust Example 🚧 - -[source,rust] ----- -use lith::{Database, Provenance}; - -#[tokio::main] -async fn main() -> Result<(), lith::Error> { - let db = Database::open("mydb/")?; - - let result = db.query( - "SELECT * FROM evidence WHERE score >= 70", - Provenance::new("user:rust-app@example.com", "API query") - ).await?; - - for doc in result.documents() { - println!("{}: {}", doc.id(), doc.get::("title")?); - } - - Ok(()) -} ----- - -=== Python Example 🚧 - -[source,python] ----- -import lith - -db = lith.open("mydb/") - -# Query -results = db.query( - "SELECT * FROM evidence WHERE score >= 70", - provenance=lith.Provenance( - actor="user:python-app@example.com", - rationale="Data analysis script" - ) -) - -for doc in results: - print(f"{doc['_id']}: {doc['title']}") - -# Insert -doc_id = db.insert( - "evidence", - {"title": "New Evidence", "score": 85}, - provenance=lith.Provenance( - actor="user:python-app@example.com", - rationale="Automated import" - ) -) ----- - -=== JavaScript Example 🚧 - -[source,javascript] ----- -import { Lith } from '@lith/client'; - -const db = await Lith.connect('https://lith.example.com', { - apiKey: process.env.LITH_API_KEY -}); - -// Query -const results = await db.query( - 'SELECT * FROM evidence WHERE score >= 70', - { - actor: 'user:js-app@example.com', - rationale: 'Dashboard refresh' - } -); - -for (const doc of results) { - console.log(`${doc._id}: ${doc.title}`); -} - -// Insert -const { id } = await db.insert('evidence', { - title: 'New Evidence', - score: 85 -}, { - actor: 'user:js-app@example.com', - rationale: 'User submission' -}); ----- - -== Status Codes Reference - -=== Success Codes - -[cols="1,1,2"] -|=== -| Code | Name | Description - -| 0 -| `LITH_OK` -| Operation completed successfully -|=== - -=== Client Errors (1xxx) - -[cols="1,1,2"] -|=== -| Code | Name | Description - -| 1001 -| `LITH_ERR_NOT_FOUND` -| Document, collection, or resource not found - -| 1002 -| `LITH_ERR_ALREADY_EXISTS` -| Resource already exists (duplicate key) - -| 1003 -| `LITH_ERR_CONSTRAINT_VIOLATION` -| Constraint check failed - -| 1004 -| `LITH_ERR_PARSE_ERROR` -| GQL syntax error - -| 1005 -| `LITH_ERR_INVALID_ARGUMENT` -| Invalid parameter value - -| 1006 -| `LITH_ERR_TYPE_MISMATCH` -| Type conversion error - -| 1007 -| `LITH_ERR_MISSING_PROVENANCE` -| Provenance required but not provided - -| 1008 -| `LITH_ERR_PERMISSION_DENIED` -| Insufficient permissions - -| 1009 -| `LITH_ERR_COLLECTION_NOT_EMPTY` -| Cannot drop non-empty collection - -| 1010 -| `LITH_ERR_INVALID_EDGE` -| Edge references non-existent document - -| 1011 -| `LITH_ERR_QUERY_TOO_COMPLEX` -| Query exceeds complexity limits - -| 1012 -| `LITH_ERR_RESULT_TOO_LARGE` -| Result set exceeds size limits -|=== - -=== Server Errors (2xxx) - -[cols="1,1,2"] -|=== -| Code | Name | Description - -| 2001 -| `LITH_ERR_INTERNAL` -| Internal server error - -| 2002 -| `LITH_ERR_IO` -| I/O error (disk, network) - -| 2003 -| `LITH_ERR_JOURNAL_CORRUPTED` -| Journal integrity check failed - -| 2004 -| `LITH_ERR_OUT_OF_MEMORY` -| Memory allocation failed - -| 2005 -| `LITH_ERR_TIMEOUT` -| Operation timed out - -| 2006 -| `LITH_ERR_STORAGE_FULL` -| Storage capacity exceeded - -| 2007 -| `LITH_ERR_ENCRYPTION` -| Encryption/decryption failed -|=== - -=== Transaction Errors (3xxx) - -[cols="1,1,2"] -|=== -| Code | Name | Description - -| 3001 -| `LITH_ERR_TXN_CONFLICT` -| Transaction conflict (concurrent modification) - -| 3002 -| `LITH_ERR_TXN_ABORTED` -| Transaction was aborted - -| 3003 -| `LITH_ERR_TXN_NOT_ACTIVE` -| No active transaction - -| 3004 -| `LITH_ERR_TXN_TOO_LARGE` -| Transaction exceeds size limits -|=== - -== See Also - -* link:../spec/gql.adoc[GQL Specification] - Complete GQL grammar -* link:../ARCHITECTURE.adoc[Architecture Guide] - System design -* link:SECURITY-AUTH.adoc[Security Guide] - Authentication and authorization -* link:DEPLOYMENT.adoc[Deployment Guide] - Production deployment diff --git a/lithoglyph/docs/CITATIONS.adoc b/lithoglyph/docs/CITATIONS.adoc deleted file mode 100644 index 6f167bdf..00000000 --- a/lithoglyph/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {PMPL-1.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/lithoglyph/docs/DEPLOYMENT.adoc b/lithoglyph/docs/DEPLOYMENT.adoc deleted file mode 100644 index dd42e350..00000000 --- a/lithoglyph/docs/DEPLOYMENT.adoc +++ /dev/null @@ -1,1078 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Deployment Guide -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font -:sectnums: - -[.lead] -How to deploy Lith in development, staging, and production environments. - -[NOTE] -==== -**Project Status: Proof of Concept** - -Lith is in active development. This guide describes the intended deployment model. Some features are planned but not yet implemented (marked with 🚧). -==== - -toc::[] - -== Overview - -Lith can be deployed in several configurations: - -[cols="1,2,2"] -|=== -| Mode | Use Case | Components - -| **Embedded** -| Single application, local data -| Form.Blocks + Form.Model only - -| **Standalone** -| Single server, multiple clients -| Full stack without clustering - -| **Clustered** -| High availability, horizontal scale -| Full stack with Elixir control plane - -| **Serverless** 🚧 -| Event-driven, auto-scaling -| Cloud-native with object storage -|=== - -== Development Setup - -=== Prerequisites - -[cols="1,1,2,1"] -|=== -| Tool | Version | Purpose | Required - -| https://github.com/cisco/gforth[Gforth] -| 0.7.3+ -| Form.Blocks, Form.Model (truth core) -| Yes - -| https://factorcode.org/[Factor] -| 0.99+ -| Form.Runtime (GQL parser/executor) -| Yes - -| https://ziglang.org/[Zig] -| 0.11+ -| Form.Bridge (stable ABI) -| Yes - -| https://just.systems/[just] -| 1.0+ -| Build orchestration -| Yes - -| https://elixir-lang.org/[Elixir] -| 1.15+ -| Form.ControlPlane (clustering) -| Optional - -| https://www.docker.com/[Docker] -| 24.0+ -| Container deployment -| Optional - -| https://leanprover.github.io/[Lean 4] -| 4.0+ -| FQLdt proofs (experimental) -| Optional -|=== - -=== Installation - -.Fedora/RHEL -[source,bash] ----- -# Core dependencies -sudo dnf install gforth - -# Zig (manual install) -curl -LO https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz -tar xf zig-linux-x86_64-0.11.0.tar.xz -sudo mv zig-linux-x86_64-0.11.0 /opt/zig -echo 'export PATH=$PATH:/opt/zig' >> ~/.bashrc - -# Just -curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin - -# Factor (manual install from factorcode.org) - -# Optional: Elixir for control plane -sudo dnf install elixir erlang ----- - -.Ubuntu/Debian -[source,bash] ----- -sudo apt update -sudo apt install gforth - -# Zig, Just, Factor: see Fedora instructions or use official sources - -# Optional: Elixir -sudo apt install elixir erlang-dev ----- - -.macOS -[source,bash] ----- -brew install gforth zig just elixir -# Factor: download from factorcode.org ----- - -.NixOS / Nix -[source,nix] ----- -# shell.nix -{ pkgs ? import {} }: -pkgs.mkShell { - buildInputs = with pkgs; [ - gforth - zig - just - factor-lang - elixir - erlang - ]; -} ----- - -=== Clone and Verify - -[source,bash] ----- -git clone https://github.com/hyperpolymath/lith.git -cd lith - -# Verify Gforth loads the core -gforth core-forth/src/lith-blocks.fs -e "cr .\" OK\" cr bye" - -# Run tests (when available) -just test ----- - -=== Development Server - -[source,bash] ----- -# Start development server with hot reload -just dev - -# Or manually: -# 1. Start the Forth core -gforth core-forth/src/lith-blocks.fs \ - core-forth/src/lith-journal.fs \ - core-forth/src/lith-model.fs - -# 2. Start the Factor runtime (when available) -factor -run=lith.runtime - -# 3. Optional: Start Elixir control plane -cd control-plane && mix phx.server ----- - -== Standalone Deployment - -Single-server deployment for small to medium workloads. - -=== Architecture - -[source,text] ----- -┌─────────────────────────────────────────────────────────────┐ -│ Client Applications │ -│ (GQL queries via HTTP/gRPC/Zig FFI) │ -└─────────────────────────┬───────────────────────────────────┘ - │ -┌─────────────────────────▼───────────────────────────────────┐ -│ Form.ControlPlane (Elixir/OTP) - Optional │ -│ ├── HTTP/gRPC API endpoints │ -│ ├── Connection pooling │ -│ ├── Session management │ -│ └── Metrics collection │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Runtime (Factor) │ -│ ├── GQL parser and planner │ -│ ├── Query execution │ -│ └── Introspection and explain │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Bridge (Zig) │ -│ ├── Stable ABI for FFI │ -│ └── Safety governor │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Model (Forth) │ -│ ├── Document collections │ -│ ├── Edge collections │ -│ └── Schema/constraint management │ -├─────────────────────────────────────────────────────────────┤ -│ Form.Blocks (Forth) │ -│ ├── 4 KiB block storage │ -│ ├── Append-only journal │ -│ └── Integrity verification │ -├─────────────────────────────────────────────────────────────┤ -│ Storage Layer │ -│ └── Local SSD / NVMe (recommended) │ -└─────────────────────────────────────────────────────────────┘ ----- - -=== Configuration - -Lith uses a hierarchical configuration system: - -[source,text] ----- -Priority (highest to lowest): -1. Environment variables (LITH_*) -2. Command-line flags -3. Config file (lith.toml or lith.ncl) -4. Built-in defaults ----- - -==== Configuration File - -.lith.toml -[source,toml] ----- -[server] -bind = "0.0.0.0" -port = 5432 -max_connections = 100 - -[storage] -data_dir = "/var/lib/lith/data" -journal_dir = "/var/lib/lith/journal" -block_size = 4096 # Do not change after init - -[journal] -sync_mode = "fsync" # fsync, fdatasync, async -checkpoint_interval = "5m" -max_segment_size = "256MB" - -[limits] -max_document_size = "16MB" -max_query_time = "30s" -max_result_size = "100MB" - -[security] -require_provenance = true -min_rationale_length = 10 - -[logging] -level = "info" # trace, debug, info, warn, error -format = "json" -output = "/var/log/lith/lith.log" - -[metrics] -enabled = true -port = 9090 -path = "/metrics" ----- - -==== Environment Variables - -[cols="2,1,2"] -|=== -| Variable | Default | Description - -| `LITH_DATA_DIR` -| `./data` -| Data directory path - -| `LITH_JOURNAL_DIR` -| `./journal` -| Journal directory path - -| `LITH_BIND` -| `127.0.0.1` -| Bind address - -| `LITH_PORT` -| `5432` -| Listen port - -| `LITH_LOG_LEVEL` -| `info` -| Log verbosity - -| `LITH_SYNC_MODE` -| `fsync` -| Journal sync mode - -| `LITH_MAX_CONNECTIONS` -| `100` -| Maximum client connections - -| `LITH_REQUIRE_PROVENANCE` -| `true` -| Require provenance on mutations -|=== - -=== Directory Structure - -[source,text] ----- -/var/lib/lith/ -├── data/ -│ ├── superblock.lgh # Database metadata -│ ├── collections/ -│ │ ├── evidence.lgh # Collection blocks -│ │ └── claims.lgh -│ └── edges/ -│ └── supports.lgh # Edge blocks -├── journal/ -│ ├── 000001.journal # Journal segments -│ ├── 000002.journal -│ └── checkpoint.meta # Last checkpoint -├── schemas/ -│ └── v1/ # Schema versions -└── config/ - └── lith.toml # Configuration ----- - -=== Systemd Service - -./etc/systemd/system/lith.service -[source,ini] ----- -[Unit] -Description=Lith Database Server -Documentation=https://github.com/hyperpolymath/lith -After=network.target - -[Service] -Type=notify -User=lith -Group=lith -ExecStart=/usr/local/bin/lith serve --config /etc/lith/lith.toml -ExecReload=/bin/kill -HUP $MAINPID -Restart=on-failure -RestartSec=5s - -# Security hardening -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/var/lib/lith /var/log/lith -PrivateTmp=true - -# Resource limits -LimitNOFILE=65536 -LimitNPROC=4096 - -[Install] -WantedBy=multi-user.target ----- - -Enable and start: - -[source,bash] ----- -sudo systemctl daemon-reload -sudo systemctl enable lith -sudo systemctl start lith -sudo systemctl status lith ----- - -== Resource Sizing - -=== Minimum Requirements - -[cols="1,1,2"] -|=== -| Resource | Minimum | Notes - -| CPU -| 2 cores -| Single-threaded Forth core + Factor runtime - -| RAM -| 4 GB -| 2 GB for runtime, 2 GB for OS/buffers - -| Storage -| 20 GB SSD -| SSD required, HDD not supported - -| Network -| 100 Mbps -| For client connections -|=== - -=== Recommended (Production) - -[cols="1,1,2"] -|=== -| Resource | Recommended | Notes - -| CPU -| 8+ cores -| Parallelism in control plane - -| RAM -| 32 GB -| More RAM = larger buffer cache - -| Storage -| 500 GB NVMe -| Low latency critical for journal - -| Network -| 1 Gbps+ -| For replication and client traffic -|=== - -=== Sizing Guidelines - -[cols="1,1,1,1"] -|=== -| Workload | Documents | Daily Writes | Recommended Spec - -| Small -| < 100K -| < 10K -| 2 CPU, 4 GB, 50 GB SSD - -| Medium -| 100K - 1M -| 10K - 100K -| 4 CPU, 16 GB, 200 GB NVMe - -| Large -| 1M - 10M -| 100K - 1M -| 8 CPU, 64 GB, 1 TB NVMe - -| Enterprise -| 10M+ -| 1M+ -| 16+ CPU, 128+ GB, RAID NVMe -|=== - -=== Storage Calculations - -[source,text] ----- -Estimated storage per document: - Block overhead: 64 bytes (header) - Average document: ~500 bytes - Journal entry: ~200 bytes - Total per document: ~764 bytes - -For 1 million documents: - Data: ~500 MB - Journal: ~200 MB (grows with writes) - Indexes: ~100 MB - Total: ~800 MB + journal history - -Journal retention: - Keep all: Unlimited audit trail - Keep 90 days: Typical for compliance - Keep 7 days: Minimal (not recommended) ----- - -== Docker Deployment - -=== Docker Image 🚧 - -[source,dockerfile] ----- -# Dockerfile -FROM debian:bookworm-slim AS builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - gforth \ - curl \ - xz-utils \ - && rm -rf /var/lib/apt/lists/* - -# Install Zig -RUN curl -LO https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz \ - && tar xf zig-linux-x86_64-0.11.0.tar.xz \ - && mv zig-linux-x86_64-0.11.0 /opt/zig - -# Copy source and build -WORKDIR /app -COPY . . -RUN PATH=$PATH:/opt/zig just build - -# Runtime image -FROM debian:bookworm-slim - -RUN apt-get update && apt-get install -y gforth && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /app/target/lith /usr/local/bin/ -COPY --from=builder /app/core-forth /opt/lith/core-forth - -# Create lith user -RUN useradd -r -s /bin/false lith \ - && mkdir -p /var/lib/lith /var/log/lith \ - && chown -R lith:lith /var/lib/lith /var/log/lith - -USER lith -WORKDIR /var/lib/lith - -EXPOSE 5432 9090 - -VOLUME ["/var/lib/lith"] - -ENTRYPOINT ["lith"] -CMD ["serve"] ----- - -=== Docker Compose - -.docker-compose.yml -[source,yaml] ----- -version: '3.8' - -services: - lith: - image: hyperpolymath/lith:latest - container_name: lith - ports: - - "5432:5432" # Main port - - "9090:9090" # Metrics - volumes: - - lith-data:/var/lib/lith - - ./lith.toml:/etc/lith/lith.toml:ro - environment: - - LITH_LOG_LEVEL=info - - LITH_SYNC_MODE=fsync - restart: unless-stopped - healthcheck: - test: ["CMD", "lith", "health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - -volumes: - lith-data: - driver: local ----- - -Run: - -[source,bash] ----- -docker-compose up -d -docker-compose logs -f lith ----- - -=== Docker Commands - -[source,bash] ----- -# Run standalone -docker run -d \ - --name lith \ - -p 5432:5432 \ - -v lith-data:/var/lib/lith \ - hyperpolymath/lith:latest - -# Execute GQL query -docker exec lith lith query 'SELECT * FROM evidence;' - -# Backup -docker exec lith lith backup /backup -docker cp lith:/backup ./lith-backup-$(date +%Y%m%d) - -# View logs -docker logs -f lith - -# Health check -docker exec lith lith health ----- - -== Kubernetes Deployment 🚧 - -=== Helm Chart - -[source,bash] ----- -# Add Helm repository -helm repo add hyperpolymath https://charts.hyperpolymath.dev -helm repo update - -# Install Lith -helm install lith hyperpolymath/lith \ - --namespace lith \ - --create-namespace \ - --set persistence.size=100Gi \ - --set resources.requests.memory=8Gi - -# Upgrade -helm upgrade lith hyperpolymath/lith \ - --namespace lith \ - --set image.tag=0.1.0 ----- - -=== Kubernetes Manifests - -.lith-deployment.yaml -[source,yaml] ----- -apiVersion: v1 -kind: Namespace -metadata: - name: lith ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: lith-data - namespace: lith -spec: - accessModes: - - ReadWriteOnce - storageClassName: fast-ssd # Use SSD storage class - resources: - requests: - storage: 100Gi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: lith - namespace: lith -spec: - serviceName: lith - replicas: 1 # Single node for standalone - selector: - matchLabels: - app: lith - template: - metadata: - labels: - app: lith - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: lith - image: hyperpolymath/lith:latest - ports: - - containerPort: 5432 - name: lith - - containerPort: 9090 - name: metrics - volumeMounts: - - name: data - mountPath: /var/lib/lith - - name: config - mountPath: /etc/lith - resources: - requests: - cpu: "2" - memory: 8Gi - limits: - cpu: "4" - memory: 16Gi - livenessProbe: - exec: - command: ["lith", "health"] - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - exec: - command: ["lith", "health", "--ready"] - initialDelaySeconds: 5 - periodSeconds: 5 - volumes: - - name: data - persistentVolumeClaim: - claimName: lith-data - - name: config - configMap: - name: lith-config ---- -apiVersion: v1 -kind: Service -metadata: - name: lith - namespace: lith -spec: - selector: - app: lith - ports: - - port: 5432 - targetPort: 5432 - name: lith - - port: 9090 - targetPort: 9090 - name: metrics ----- - -== Cloud Provider Guides - -=== AWS - -==== EC2 Deployment - -[source,bash] ----- -# Recommended instance types -# - Development: t3.medium (2 vCPU, 4 GB) -# - Production: r6i.xlarge (4 vCPU, 32 GB) -# - High-performance: r6i.4xlarge (16 vCPU, 128 GB) - -# Use gp3 or io2 EBS volumes for storage -# Minimum IOPS: 3000 (gp3 default) -# Recommended: 10000+ IOPS for production ----- - -==== EBS Configuration - -[source,bash] ----- -# Create optimized EBS volume -aws ec2 create-volume \ - --availability-zone us-east-1a \ - --size 500 \ - --volume-type gp3 \ - --iops 10000 \ - --throughput 500 \ - --tag-specifications 'ResourceType=volume,Tags=[{Key=Name,Value=lith-data}]' ----- - -==== S3 for Backups - -[source,bash] ----- -# Backup to S3 -lith backup /tmp/backup -aws s3 sync /tmp/backup s3://my-bucket/lith-backups/$(date +%Y%m%d)/ - -# Restore from S3 -aws s3 sync s3://my-bucket/lith-backups/20260112/ /tmp/restore -lith restore /tmp/restore ----- - -=== Google Cloud Platform - -==== Compute Engine - -[source,bash] ----- -# Recommended machine types -# - Development: e2-medium (2 vCPU, 4 GB) -# - Production: n2-highmem-4 (4 vCPU, 32 GB) -# - High-performance: n2-highmem-16 (16 vCPU, 128 GB) - -# Create VM with SSD -gcloud compute instances create lith-server \ - --machine-type=n2-highmem-4 \ - --zone=us-central1-a \ - --boot-disk-size=50GB \ - --boot-disk-type=pd-ssd \ - --create-disk=name=lith-data,size=500GB,type=pd-ssd,auto-delete=no ----- - -==== Cloud Storage for Backups - -[source,bash] ----- -# Backup to GCS -lith backup /tmp/backup -gsutil -m rsync -r /tmp/backup gs://my-bucket/lith-backups/$(date +%Y%m%d)/ ----- - -=== Azure - -==== Virtual Machines - -[source,bash] ----- -# Recommended VM sizes -# - Development: Standard_D2s_v3 (2 vCPU, 8 GB) -# - Production: Standard_E4s_v3 (4 vCPU, 32 GB) -# - High-performance: Standard_E16s_v3 (16 vCPU, 128 GB) - -# Create VM with Premium SSD -az vm create \ - --resource-group lith-rg \ - --name lith-server \ - --size Standard_E4s_v3 \ - --image Debian:debian-12:12:latest \ - --data-disk-sizes-gb 500 \ - --storage-sku Premium_LRS ----- - -== High Availability 🚧 - -=== Clustered Architecture - -[source,text] ----- - ┌─────────────────┐ - │ Load Balancer │ - └────────┬────────┘ - │ - ┌───────────────────┼───────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Lith Node 1 │ │ Lith Node 2 │ │ Lith Node 3 │ -│ (Primary) │ │ (Replica) │ │ (Replica) │ -│ │ │ │ │ │ -│ Control Plane │ │ Control Plane │ │ Control Plane │ -│ ───────────── │ │ ───────────── │ │ ───────────── │ -│ Runtime │ │ Runtime │ │ Runtime │ -│ ───────────── │ │ ───────────── │ │ ───────────── │ -│ Storage │ │ Storage │ │ Storage │ -└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ - │ │ │ - └───────────────────┴───────────────────┘ - │ - ┌────────▼────────┐ - │ Shared Journal │ - │ (Raft/Paxos) │ - └─────────────────┘ ----- - -=== Replication Modes - -[cols="1,2,2"] -|=== -| Mode | Description | Use Case - -| **Synchronous** -| All replicas confirm before commit -| Maximum durability, highest latency - -| **Asynchronous** -| Primary confirms immediately -| Lower latency, potential data loss - -| **Semi-synchronous** -| At least one replica confirms -| Balance of durability and latency -|=== - -== Backup and Recovery - -=== Backup Strategy - -[source,bash] ----- -# Full backup -lith backup --full /backup/full-$(date +%Y%m%d) - -# Incremental backup (journal segments since last backup) -lith backup --incremental --since 1000 /backup/incr-$(date +%Y%m%d) - -# Continuous backup (stream journal to remote) -lith backup --stream s3://my-bucket/lith-journal/ ----- - -=== Backup Schedule - -[cols="1,2,2"] -|=== -| Type | Frequency | Retention - -| Full backup -| Weekly (Sunday 02:00) -| 4 weeks - -| Incremental -| Daily (02:00) -| 7 days - -| Journal stream -| Continuous -| 90 days -|=== - -=== Recovery - -[source,bash] ----- -# Restore from full backup -lith restore /backup/full-20260112 - -# Restore to point in time -lith restore /backup/full-20260112 --to-sequence 5000 - -# Restore to timestamp -lith restore /backup/full-20260112 --to-time "2026-01-12T10:30:00Z" - -# Verify restoration -lith doctor --verify ----- - -== Upgrades - -=== Upgrade Process - -[source,bash] ----- -# 1. Check compatibility -lith doctor --check-upgrade --to-version 0.2.0 - -# 2. Backup current state -lith backup --full /backup/pre-upgrade-$(date +%Y%m%d) - -# 3. Stop service -sudo systemctl stop lith - -# 4. Install new version -# (package manager or manual) - -# 5. Run migrations if needed -lith migrate --to-version 0.2.0 - -# 6. Start service -sudo systemctl start lith - -# 7. Verify -lith doctor --verify -lith query 'INTROSPECT VERSION;' ----- - -=== Rollback - -[source,bash] ----- -# If upgrade fails -sudo systemctl stop lith - -# Restore previous version binary -# ... - -# Restore from backup -lith restore /backup/pre-upgrade-20260112 - -sudo systemctl start lith ----- - -== Health Checks - -=== CLI Health Check - -[source,bash] ----- -# Basic health -lith health - -# Detailed health -lith health --verbose - -# Output: -# Lith Health Check -# =================== -# Status: HEALTHY -# -# Components: -# Storage: OK (500 GB free) -# Journal: OK (1523 entries, synced) -# Runtime: OK (uptime: 5d 12h) -# Connections: OK (12/100) -# -# Last checkpoint: 2026-01-12T10:00:00Z -# Journal lag: 0 entries ----- - -=== HTTP Health Endpoints - -[cols="1,1,2"] -|=== -| Endpoint | Response | Purpose - -| `GET /health` -| 200 OK -| Basic liveness - -| `GET /health/ready` -| 200 OK / 503 -| Readiness (can accept queries) - -| `GET /health/live` -| 200 OK / 503 -| Liveness (process running) - -| `GET /health/storage` -| JSON status -| Storage subsystem health - -| `GET /health/journal` -| JSON status -| Journal subsystem health -|=== - -== Troubleshooting - -=== Common Issues - -==== "Permission denied" on data directory - -[source,bash] ----- -# Fix ownership -sudo chown -R lith:lith /var/lib/lith - -# Fix permissions -sudo chmod -R 750 /var/lib/lith ----- - -==== "Journal sync failed" - -[source,bash] ----- -# Check disk space -df -h /var/lib/lith - -# Check disk I/O -iostat -x 1 - -# Switch to async mode temporarily (data loss risk!) -LITH_SYNC_MODE=async lith serve ----- - -==== "Too many open files" - -[source,bash] ----- -# Check current limit -ulimit -n - -# Increase limit -sudo bash -c 'echo "lith soft nofile 65536" >> /etc/security/limits.conf' -sudo bash -c 'echo "lith hard nofile 65536" >> /etc/security/limits.conf' - -# For systemd service, add to [Service]: -# LimitNOFILE=65536 ----- - -==== "Out of memory" - -[source,bash] ----- -# Check memory usage -free -h -ps aux | grep lith - -# Reduce buffer cache (in lith.toml) -# [storage] -# buffer_cache_size = "1GB" ----- - -== See Also - -* link:OBSERVABILITY.adoc[Observability Guide] - Monitoring and metrics -* link:SECURITY-AUTH.adoc[Security & Auth] - Authentication and hardening -* link:../ARCHITECTURE.adoc[Architecture Guide] - System design -* link:../VERSIONING.adoc[Versioning Policy] - Upgrade compatibility diff --git a/lithoglyph/docs/EPSTEIN-INGEST-TESTS.md b/lithoglyph/docs/EPSTEIN-INGEST-TESTS.md deleted file mode 100644 index ce7b7fa5..00000000 --- a/lithoglyph/docs/EPSTEIN-INGEST-TESTS.md +++ /dev/null @@ -1,133 +0,0 @@ -# Epstein Files — Lithoglyph Ingest Tests & Benchmarks -# -# SPDX-License-Identifier: CC-BY-SA-4.0 -# Author: Jonathan D.A. Jewell -# Created: 2026-03-13 -# -# Extracted from the master pathway: bofig/docs/EPSTEIN-FILES-WORK-PATHWAY.md -# This file contains only the Lithoglyph-specific phases (2.1–2.5, 4.4, 5.2). - -## Phase 2: Lithoglyph Ingest & Storage (Weeks 4–10) - -### Step 2.1: Zig 0.15.2 HTTP API Migration (L1) - -83 call sites need updating for the new Zig HTTP API. - -**Tests:** -- [ ] T-ZIG-1: All 83 call sites compile with zig 0.15.2 -- [ ] T-ZIG-2: HTTP server starts and responds to GET /health -- [ ] T-ZIG-3: GQL INSERT via HTTP returns 200 + created record ID -- [ ] T-ZIG-4: Concurrent 100-request stress test — no crashes - -**Benchmarks:** -- [ ] B-ZIG-1: GQL INSERT latency — target: <5ms p99 -- [ ] B-ZIG-2: Batch INSERT (1000 records) — target: <500ms total - -### Step 2.2: Evidence Collection Schema (L3) - -Collections: `bofig_evidence`, `bofig_claims`, `bofig_relationships` - -**Tests:** -- [ ] T-EVD-1: CREATE bofig_evidence collection succeeds -- [ ] T-EVD-2: INSERT evidence record with all PROMPT dimensions -- [ ] T-EVD-3: QUERY evidence by SHA-256 hash (dedup lookup) -- [ ] T-EVD-4: QUERY evidence by entity name (cross-reference) -- [ ] T-EVD-5: All mutations have actor + rationale (Lithoglyph invariant) - -### Step 2.3: Financial Transaction Collection (L4) - -Schema: source, destination, amount, currency, date, instrument, intermediary - -**Tests:** -- [ ] T-FTX-1: INSERT transaction record with full fields -- [ ] T-FTX-2: QUERY transaction chain (A→B→C) via GQL path traversal -- [ ] T-FTX-3: Aggregate: total flow between two entities -- [ ] T-FTX-4: Temporal: transactions within date range -- [ ] T-FTX-5: Anomaly: detect round-number patterns (e.g., exactly $10,000) - -**Benchmarks:** -- [ ] B-FTX-1: 16,000 transaction inserts — target: <10 seconds -- [ ] B-FTX-2: Transaction chain query (depth 5) — target: <100ms - -### Step 2.4: Entity Collection + Co-Reference Resolution (L5) - -Alias tracking: "Jeffrey Epstein" = "J. Epstein" = "Epstein, Jeffrey" - -**Tests:** -- [ ] T-ENT-1: CREATE entity with primary name -- [ ] T-ENT-2: ADD alias to existing entity -- [ ] T-ENT-3: MERGE two entities (logged in journal with rationale) -- [ ] T-ENT-4: REVERSE merge (undo co-reference error) -- [ ] T-ENT-5: QUERY all documents mentioning entity (across aliases) -- [ ] T-ENT-6: No orphaned aliases after merge/unmerge cycle - -**Benchmarks:** -- [ ] B-ENT-1: 23,000 entity inserts with alias resolution — target: <60 seconds -- [ ] B-ENT-2: Entity lookup by any alias — target: <10ms - -### Step 2.5: Docudactyl → Lithoglyph Ingest Bridge (D2 + L6) - -Cap'n Proto → GQL INSERT with auto-PROMPT scoring. - -**Tests:** -- [ ] T-BRG-1: Single Cap'n Proto StageResults → Lithoglyph evidence record -- [ ] T-BRG-2: PROMPT auto-scoring from extraction metadata: - - OCR confidence 90+ → Provenance score 0.8+ - - Multiple corroborating documents → Replicability score increases - - Court filing (official source) → Publication score 0.9+ -- [ ] T-BRG-3: SHA-256 dedup: duplicate document skipped with log -- [ ] T-BRG-4: Batch import 1000 records — all arrive with provenance -- [ ] T-BRG-5: Actor="docudactyl-pipeline", Rationale includes run ID - -**Benchmarks:** -- [ ] B-BRG-1: 10,000 records batch import — target: <30 seconds -- [ ] B-BRG-2: 3.2M records full import — target: <6 hours - ---- - -## Phase 4: Temporal Credibility Model (L7) (Weeks 12–18) - -Source reputation evolving over time. - -**Tests:** -- [ ] T-TCR-1: New source starts at neutral credibility -- [ ] T-TCR-2: Source's claim independently verified → credibility increases -- [ ] T-TCR-3: Source caught in contradiction → credibility decreases -- [ ] T-TCR-4: Source retraction → credibility impact + retraction logged -- [ ] T-TCR-5: Time-travel: "What was this source's credibility on 2023-01-15?" -- [ ] T-TCR-6: Credibility affects PROMPT scores of all evidence from that source - ---- - -## Phase 5: Cross-Investigation Linking (L8) (Weeks 16–22) - -Shared evidence across investigations (Epstein ↔ Maxwell ↔ related cases). - -**Tests:** -- [ ] T-XIL-1: Evidence in investigation A also relevant to investigation B → linked -- [ ] T-XIL-2: Entity appearing in both investigations → surfaced automatically -- [ ] T-XIL-3: New investigation inherits relevant evidence from existing investigations -- [ ] T-XIL-4: Access controls per investigation - ---- - -## Completion Tracker (Lithoglyph Phases Only) - -| # | Step | Status | Tests | Benchmarks | -|---|------|--------|-------|------------| -| 2.1 | Zig API Migration | **DONE** | 0/4 | 0/2 | -| 2.2 | Evidence Schema | **DONE** | 0/5 | — | -| 2.3 | Financial Txn Collection | TODO | 0/5 | 0/2 | -| 2.4 | Entity + Co-Ref | TODO | 0/6 | 0/2 | -| 2.5 | Ingest Bridge | TODO | 0/5 | 0/2 | -| 4.4 | Temporal Credibility | TODO | 0/6 | — | -| 5.2 | Cross-Investigation | TODO | 0/4 | — | - -**Totals: 35 tests, 8 benchmarks | Current: 0 tests written, 0 benchmarks run** - -## Cross-References - -- **Full pipeline pathway:** `bofig/docs/EPSTEIN-FILES-WORK-PATHWAY.md` -- **Master integration plan:** `bofig/docs/INTEGRATION-PLAN.md` -- **Lithoglyph integration role:** `docs/INTEGRATION-PLAN-LITHOGLYPH.md` (this repo) -- **GQL dependent types spec:** `spec/gql-dependent-types.md` (this repo) diff --git a/lithoglyph/docs/INTEGRATION-PATTERNS.adoc b/lithoglyph/docs/INTEGRATION-PATTERNS.adoc deleted file mode 100644 index 6294670a..00000000 --- a/lithoglyph/docs/INTEGRATION-PATTERNS.adoc +++ /dev/null @@ -1,1634 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Integration Patterns -:toc: left -:toclevels: 4 -:icons: font -:source-highlighter: rouge -:sectanchors: -:sectlinks: - -[.lead] -Comprehensive guide to integrating Lith with message queues, search engines, analytics platforms, AI/ML pipelines, and other external systems while maintaining audit-first guarantees. - -[NOTE] -==== -**Status: Documentation Complete, Adapters Planned** - -This document provides complete integration patterns. Native adapters are planned for Milestone M11. Currently, integrations use the documented patterns with GQL API or HTTP REST interface. -==== - -== Integration Philosophy - -Lith integrations follow core principles: - -[cols="1,3"] -|=== -| Principle | Implementation - -| **Provenance Preservation** -| External system references captured in provenance (Kafka offset, S3 ETag) - -| **Journal as Source** -| All external sync derives from journal, enabling replay and recovery - -| **Exactly-Once Semantics** -| Sequence numbers enable idempotent processing - -| **Audit Trail Extension** -| External system actions logged with Lith provenance -|=== - -=== Integration Patterns Overview - -[source] ----- - ┌──────────────────────────────────────────────────────────┐ - │ Lith Core │ - │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ - │ │ Journal │ │ Collections │ │ Edges │ │ - │ └─────────────┘ └─────────────┘ └─────────────┘ │ - └────────┬───────────────────────────────────────────────┘ - │ - ┌───────────┼───────────┬───────────────┬──────────────┐ - │ │ │ │ │ - ▼ ▼ ▼ ▼ ▼ - ┌────────────┐ ┌────────┐ ┌─────────┐ ┌────────────┐ ┌────────────┐ - │ Message │ │ Search │ │Analytics│ │ Graph │ │ AI/ML │ - │ Queues │ │Engines │ │ & BI │ │ Databases │ │ Pipelines │ - └────────────┘ └────────┘ └─────────┘ └────────────┘ └────────────┘ ----- - -== Message Queues - -=== Kafka Integration - -==== Inbound: Kafka → Lith - -Consume Kafka messages and store with provenance tracking: - -[source] ----- -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Producers │────▶│ Kafka │────▶│ Lith │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ - ▼ - [Journal with - Kafka offset - in provenance] ----- - -.Kafka Consumer Configuration -[source,toml] ----- -[integrations.kafka.inbound] -enabled = true -bootstrap_servers = ["kafka-1:9092", "kafka-2:9092", "kafka-3:9092"] -group_id = "lith-consumer" -topics = ["events", "documents", "updates"] - -# Auto-commit disabled for exactly-once semantics -auto_commit = false -# Commit after Lith journal write confirmed -commit_strategy = "after_journal" - -# Message to GQL mapping -[integrations.kafka.inbound.mapping] -topic = "events" -collection = "events" -key_field = "event_id" - -# Provenance from Kafka metadata -[integrations.kafka.inbound.provenance] -actor_template = "kafka:${topic}:${partition}" -rationale_template = "Kafka message offset=${offset} timestamp=${timestamp}" ----- - -.Consumer Implementation Example -[source,javascript] ----- -// Kafka → Lith consumer with exactly-once semantics -import { Kafka } from 'kafkajs'; -import { LithClient } from 'lith-client'; - -const kafka = new Kafka({ brokers: ['kafka:9092'] }); -const consumer = kafka.consumer({ groupId: 'lith-consumer' }); -const lith = new LithClient('lith://localhost:8765'); - -await consumer.connect(); -await consumer.subscribe({ topic: 'events', fromBeginning: false }); - -await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - const event = JSON.parse(message.value.toString()); - - // Insert with Kafka provenance - await lith.query(` - INSERT INTO events $1 - WITH PROVENANCE { - actor: $2, - rationale: $3 - } - `, [ - event, - `kafka:${topic}:${partition}`, - `offset=${message.offset} key=${message.key}` - ]); - - // Commit only after successful insert - await consumer.commitOffsets([{ - topic, - partition, - offset: (parseInt(message.offset) + 1).toString() - }]); - } -}); ----- - -==== Outbound: Lith → Kafka - -Publish journal changes to Kafka topics: - -.CDC to Kafka Configuration -[source,toml] ----- -[integrations.kafka.outbound] -enabled = true -bootstrap_servers = ["kafka-1:9092", "kafka-2:9092", "kafka-3:9092"] -client_id = "lith-publisher" - -# Topic per collection -topic_template = "lith.${collection}" -# Or single topic -# topic = "lith-changes" - -# What to publish -include_collections = ["*"] # All collections -exclude_collections = ["_internal"] - -# Message format -[integrations.kafka.outbound.format] -type = "json" # json, avro, protobuf -include_provenance = true -include_before = true # For updates -include_after = true - -# Delivery guarantees -[integrations.kafka.outbound.delivery] -acks = "all" -retries = 3 -idempotent = true ----- - -.Outbound Message Format -[source,json] ----- -{ - "sequence": 100542, - "timestamp": "2026-01-12T10:30:00Z", - "operation": "INSERT", - "collection": "evidence", - "key": "doc_12345", - "before": null, - "after": { - "_key": "doc_12345", - "title": "Leaked memo", - "source": "anonymous", - "score": 85 - }, - "provenance": { - "actor": "user:alice@session:xyz", - "rationale": "New evidence from whistleblower" - } -} ----- - -=== RabbitMQ Integration - -.RabbitMQ Configuration -[source,toml] ----- -[integrations.rabbitmq] -enabled = true -uri = "amqp://user:pass@rabbitmq:5672/lith" - -[integrations.rabbitmq.inbound] -queue = "lith-inbound" -exchange = "documents" -routing_key = "document.*" -prefetch = 100 - -[integrations.rabbitmq.outbound] -exchange = "lith-changes" -exchange_type = "topic" -routing_key_template = "${collection}.${operation}" ----- - -.RabbitMQ Consumer Example -[source,javascript] ----- -import amqp from 'amqplib'; -import { LithClient } from 'lith-client'; - -const connection = await amqp.connect('amqp://rabbitmq:5672'); -const channel = await connection.createChannel(); -const lith = new LithClient('lith://localhost:8765'); - -await channel.assertQueue('lith-inbound', { durable: true }); - -channel.consume('lith-inbound', async (msg) => { - const document = JSON.parse(msg.content.toString()); - - try { - await lith.insert(document.collection, document.data, { - actor: `rabbitmq:${msg.properties.messageId}`, - rationale: `RabbitMQ delivery tag=${msg.fields.deliveryTag}` - }); - - channel.ack(msg); - } catch (error) { - // Requeue on failure - channel.nack(msg, false, true); - } -}); ----- - -=== NATS Integration - -.NATS Configuration -[source,toml] ----- -[integrations.nats] -enabled = true -servers = ["nats://nats-1:4222", "nats://nats-2:4222"] - -[integrations.nats.jetstream] -enabled = true -stream = "LITHOGLYPH" -consumer = "lith-processor" - -[integrations.nats.inbound] -subjects = ["documents.>", "events.>"] - -[integrations.nats.outbound] -subject_template = "lith.changes.${collection}" ----- - -.NATS JetStream Consumer -[source,javascript] ----- -import { connect, StringCodec } from 'nats'; -import { LithClient } from 'lith-client'; - -const nc = await connect({ servers: 'nats://localhost:4222' }); -const js = nc.jetstream(); -const lith = new LithClient('lith://localhost:8765'); -const sc = StringCodec(); - -const sub = await js.subscribe('documents.*', { - durable: 'lith-processor', - ack_policy: 'explicit' -}); - -for await (const msg of sub) { - const document = JSON.parse(sc.decode(msg.data)); - - await lith.insert('documents', document, { - actor: `nats:${msg.subject}`, - rationale: `NATS seq=${msg.seq}` - }); - - msg.ack(); -} ----- - -=== Apache Pulsar Integration - -.Pulsar Configuration -[source,toml] ----- -[integrations.pulsar] -enabled = true -service_url = "pulsar://pulsar:6650" -admin_url = "http://pulsar:8080" - -[integrations.pulsar.inbound] -topics = ["persistent://tenant/namespace/documents"] -subscription = "lith-subscription" -subscription_type = "Shared" - -[integrations.pulsar.outbound] -topic = "persistent://tenant/namespace/lith-changes" -batching = true -batch_max_messages = 100 ----- - -== Search Engines - -=== Elasticsearch / OpenSearch - -==== Journal → Search Index Sync - -[source] ----- -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Lith │────▶│ CDC │────▶│Elasticsearch│ -│ Journal │ │ Connector │ │ Index │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ │ - │ ┌────┴────┐ - │ │ Offset │ - │ │ Tracker │ - │ └─────────┘ - │ │ - └───────────────────┘ - Checkpoint stored - in Lith ----- - -.Elasticsearch Sync Configuration -[source,toml] ----- -[integrations.elasticsearch] -enabled = true -hosts = ["https://es-node-1:9200", "https://es-node-2:9200"] -api_key = "${ELASTICSEARCH_API_KEY}" - -# Index naming -index_template = "lith-${collection}" - -# Field mappings -[integrations.elasticsearch.mappings.evidence] -title = { type = "text", analyzer = "english" } -content = { type = "text", analyzer = "english" } -source = { type = "keyword" } -score = { type = "integer" } -created_at = { type = "date" } -_provenance = { type = "object", enabled = false } # Don't index provenance - -# Sync settings -[integrations.elasticsearch.sync] -batch_size = 100 -flush_interval_ms = 1000 -checkpoint_collection = "_es_checkpoints" ----- - -.Elasticsearch Sync Implementation -[source,javascript] ----- -import { Client } from '@elastic/elasticsearch'; -import { LithClient } from 'lith-client'; - -const es = new Client({ node: 'https://elasticsearch:9200' }); -const lith = new LithClient('lith://localhost:8765'); - -async function syncToElasticsearch() { - // Get last synced sequence - const checkpoint = await lith.query(` - SELECT sequence FROM _es_checkpoints - WHERE index = 'evidence' - `); - const lastSeq = checkpoint[0]?.sequence || 0; - - // Get journal entries since checkpoint - const entries = await lith.query(` - SELECT * FROM _journal - WHERE sequence > $1 - AND collection = 'evidence' - ORDER BY sequence - LIMIT 1000 - `, [lastSeq]); - - if (entries.length === 0) return; - - // Bulk index to Elasticsearch - const body = entries.flatMap(entry => { - switch (entry.op_type) { - case 'DOC_INSERT': - case 'DOC_UPDATE': - return [ - { index: { _index: 'lith-evidence', _id: entry.document_key } }, - entry.data - ]; - case 'DOC_DELETE': - return [ - { delete: { _index: 'lith-evidence', _id: entry.document_key } } - ]; - default: - return []; - } - }); - - await es.bulk({ body, refresh: true }); - - // Update checkpoint - const maxSeq = Math.max(...entries.map(e => e.sequence)); - await lith.query(` - UPSERT _es_checkpoints - WHERE index = 'evidence' - SET sequence = $1 - WITH PROVENANCE { - actor: "es-sync", - rationale: "Elasticsearch sync checkpoint" - } - `, [maxSeq]); -} - -// Run sync every 5 seconds -setInterval(syncToElasticsearch, 5000); ----- - -==== Search Query Forwarding - -.Federated Search Configuration -[source,toml] ----- -[integrations.elasticsearch.search] -# Enable search through GQL -enabled = true - -# GQL SEARCH operator uses Elasticsearch -search_operator = "elasticsearch" - -# Fallback to Lith scan if ES unavailable -fallback_to_scan = true ----- - -.Using Elasticsearch Search in GQL -[source,gql] ----- --- GQL with Elasticsearch search -SELECT * FROM evidence -WHERE SEARCH("leaked memo corruption") - AND score > 70 -ORDER BY _score DESC -LIMIT 10; - --- Full-text search with filters -SELECT * FROM evidence -WHERE SEARCH("financial irregularities") - AND source IN ('internal', 'whistleblower') -WITH PROVENANCE { - actor: "investigator:jane", - rationale: "Research for Panama Papers follow-up" -}; ----- - -=== Meilisearch Integration - -.Meilisearch Configuration -[source,toml] ----- -[integrations.meilisearch] -enabled = true -host = "http://meilisearch:7700" -api_key = "${MEILISEARCH_API_KEY}" - -# Index settings -[integrations.meilisearch.indexes.evidence] -primary_key = "_key" -searchable_attributes = ["title", "content", "source"] -filterable_attributes = ["score", "source", "created_at"] -sortable_attributes = ["score", "created_at"] ----- - -.Meilisearch Sync -[source,javascript] ----- -import { MeiliSearch } from 'meilisearch'; -import { LithClient } from 'lith-client'; - -const meili = new MeiliSearch({ - host: 'http://meilisearch:7700', - apiKey: process.env.MEILISEARCH_API_KEY -}); -const lith = new LithClient('lith://localhost:8765'); - -async function syncToMeilisearch() { - const checkpoint = await getCheckpoint('meilisearch', 'evidence'); - const entries = await lith.journalSince(checkpoint, 'evidence', 500); - - const updates = entries - .filter(e => ['DOC_INSERT', 'DOC_UPDATE'].includes(e.op_type)) - .map(e => e.data); - - const deletes = entries - .filter(e => e.op_type === 'DOC_DELETE') - .map(e => e.document_key); - - if (updates.length > 0) { - await meili.index('evidence').addDocuments(updates); - } - if (deletes.length > 0) { - await meili.index('evidence').deleteDocuments(deletes); - } - - await updateCheckpoint('meilisearch', 'evidence', entries); -} ----- - -=== Typesense Integration - -.Typesense Configuration -[source,toml] ----- -[integrations.typesense] -enabled = true -nodes = [ - { host = "ts-1", port = 8108, protocol = "https" }, - { host = "ts-2", port = 8108, protocol = "https" } -] -api_key = "${TYPESENSE_API_KEY}" - -[integrations.typesense.collections.evidence] -fields = [ - { name = "title", type = "string" }, - { name = "content", type = "string" }, - { name = "source", type = "string", facet = true }, - { name = "score", type = "int32" }, - { name = "created_at", type = "int64" } -] -default_sorting_field = "score" ----- - -== Analytics & BI - -=== Data Warehouse Export - -==== Journal → Parquet → Warehouse - -[source] ----- -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Lith │────▶│ Parquet │────▶│ BigQuery/ │ -│ Journal │ │ Export │ │ Snowflake │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ - ▼ - ┌─────────────┐ - │ S3/ │ - │ GCS │ - └─────────────┘ ----- - -.Parquet Export Configuration -[source,toml] ----- -[integrations.warehouse.export] -enabled = true -format = "parquet" -compression = "snappy" - -# Export schedule -schedule = "0 * * * *" # Hourly - -# Output destination -[integrations.warehouse.export.destination] -type = "s3" -bucket = "lith-analytics" -prefix = "exports/" -partition_by = ["date", "collection"] - -# BigQuery load job -[integrations.warehouse.bigquery] -enabled = true -project = "my-project" -dataset = "lith_analytics" -table_prefix = "lith_" ----- - -.Export Script -[source,bash] ----- -#!/bin/bash -# export-to-warehouse.sh - -# Export journal to Parquet -lith export \ - --format parquet \ - --collection evidence \ - --since "$(date -d 'yesterday' +%Y-%m-%d)" \ - --until "$(date +%Y-%m-%d)" \ - --output s3://lith-analytics/exports/$(date +%Y/%m/%d)/ - -# Load to BigQuery -bq load \ - --source_format=PARQUET \ - --replace \ - my-project:lith_analytics.evidence_$(date +%Y%m%d) \ - "gs://lith-analytics/exports/$(date +%Y/%m/%d)/evidence_*.parquet" ----- - -=== Apache Arrow / DuckDB Integration - -.DuckDB Direct Query -[source,sql] ----- --- DuckDB can query Lith exports directly -SELECT - date_trunc('day', created_at) as day, - source, - COUNT(*) as count, - AVG(score) as avg_score -FROM read_parquet('s3://lith-analytics/exports/**/*.parquet') -WHERE score > 70 -GROUP BY day, source -ORDER BY day, count DESC; ----- - -.Arrow Flight Integration -[source,toml] ----- -[integrations.arrow_flight] -enabled = true -port = 8815 - -# Collections exposed via Arrow Flight -expose_collections = ["evidence", "sources", "articles"] - -# Query result caching -cache_enabled = true -cache_ttl_seconds = 300 ----- - -.Python Arrow Flight Client -[source,python] ----- -import pyarrow.flight as flight - -# Connect to Lith Arrow Flight endpoint -client = flight.connect("grpc://lith:8815") - -# Execute GQL query and get Arrow table -info = client.get_flight_info( - flight.FlightDescriptor.for_command( - b"SELECT * FROM evidence WHERE score > 80" - ) -) - -# Stream results as Arrow batches -reader = client.do_get(info.endpoints[0].ticket) -table = reader.read_all() - -# Convert to pandas -df = table.to_pandas() -print(df.describe()) ----- - -=== Apache Spark Integration - -.Spark Connector Configuration -[source,scala] ----- -// Spark DataFrame from Lith -val df = spark.read - .format("lith") - .option("host", "lith://localhost:8765") - .option("collection", "evidence") - .option("query", "SELECT * FROM evidence WHERE score > 70") - .load() - -df.groupBy("source") - .agg( - count("*").as("count"), - avg("score").as("avg_score") - ) - .write - .mode("overwrite") - .parquet("s3://analytics/evidence-summary/") ----- - -== Graph Databases - -=== Neo4j Integration - -Lith edge collections can sync to Neo4j for graph analytics. - -.Neo4j Sync Configuration -[source,toml] ----- -[integrations.neo4j] -enabled = true -uri = "bolt://neo4j:7687" -username = "${NEO4J_USER}" -password = "${NEO4J_PASSWORD}" -database = "lith" - -# Sync document collections as nodes -[integrations.neo4j.nodes] -evidence = { label = "Evidence", id_field = "_key" } -sources = { label = "Source", id_field = "_key" } -people = { label = "Person", id_field = "_key" } - -# Sync edge collections as relationships -[integrations.neo4j.relationships] -cites = { type = "CITES", from = "evidence", to = "sources" } -authored_by = { type = "AUTHORED_BY", from = "evidence", to = "people" } -mentions = { type = "MENTIONS", from = "evidence", to = "people" } ----- - -.Neo4j Sync Implementation -[source,javascript] ----- -import neo4j from 'neo4j-driver'; -import { LithClient } from 'lith-client'; - -const driver = neo4j.driver('bolt://neo4j:7687', - neo4j.auth.basic('neo4j', 'password')); -const lith = new LithClient('lith://localhost:8765'); - -async function syncEdgesToNeo4j(edgeCollection, relType) { - const checkpoint = await getCheckpoint('neo4j', edgeCollection); - const entries = await lith.journalSince(checkpoint, edgeCollection, 1000); - - const session = driver.session(); - - for (const entry of entries) { - if (entry.op_type === 'EDGE_INSERT') { - await session.run(` - MATCH (a {lith_key: $from}) - MATCH (b {lith_key: $to}) - MERGE (a)-[r:${relType} {lith_key: $key}]->(b) - SET r += $props - `, { - from: entry.data._from, - to: entry.data._to, - key: entry.data._key, - props: entry.data.properties || {} - }); - } else if (entry.op_type === 'EDGE_DELETE') { - await session.run(` - MATCH ()-[r:${relType} {lith_key: $key}]->() - DELETE r - `, { key: entry.document_key }); - } - } - - await session.close(); - await updateCheckpoint('neo4j', edgeCollection, entries); -} ----- - -.Querying Neo4j for Graph Analytics -[source,cypher] ----- -// Find shortest path between two pieces of evidence -MATCH path = shortestPath( - (a:Evidence {lith_key: 'ev_001'})-[*]-(b:Evidence {lith_key: 'ev_002'}) -) -RETURN path; - -// Find evidence connected to a person through multiple hops -MATCH (p:Person {name: 'John Doe'})<-[:MENTIONS|AUTHORED_BY*1..3]-(e:Evidence) -RETURN e, p; - -// Community detection -CALL gds.louvain.stream('evidence-graph') -YIELD nodeId, communityId -RETURN gds.util.asNode(nodeId).title, communityId -ORDER BY communityId; ----- - -=== RDF / SPARQL Integration - -.RDF Mapping Configuration -[source,toml] ----- -[integrations.rdf] -enabled = true -base_uri = "https://lith.example.org/" - -# Vocabulary prefixes -[integrations.rdf.prefixes] -lith = "https://lith.example.org/ontology#" -dc = "http://purl.org/dc/elements/1.1/" -prov = "http://www.w3.org/ns/prov#" -schema = "https://schema.org/" - -# Collection to RDF class mapping -[integrations.rdf.classes] -evidence = "lith:Evidence" -sources = "lith:Source" -people = "schema:Person" - -# Property mappings -[integrations.rdf.properties.evidence] -title = "dc:title" -content = "dc:description" -source = "dc:source" -score = "lith:promptScore" -created_at = "dc:date" -_provenance.actor = "prov:wasAttributedTo" ----- - -.SPARQL Endpoint -[source,toml] ----- -[integrations.sparql] -enabled = true -port = 8890 -endpoint = "/sparql" - -# Allow SPARQL updates (mutations) -allow_updates = false # Read-only for safety - -# Query timeout -timeout_seconds = 30 ----- - -.SPARQL Queries -[source,sparql] ----- -PREFIX lith: -PREFIX dc: -PREFIX prov: - -# Find all evidence by a specific actor -SELECT ?evidence ?title ?score -WHERE { - ?evidence a lith:Evidence ; - dc:title ?title ; - lith:promptScore ?score ; - prov:wasAttributedTo ?actor . - FILTER (?actor = "user:alice") - FILTER (?score > 70) -} -ORDER BY DESC(?score) - -# Count evidence by source -SELECT ?source (COUNT(?evidence) as ?count) -WHERE { - ?evidence a lith:Evidence ; - dc:source ?source . -} -GROUP BY ?source -ORDER BY DESC(?count) ----- - -== Event Sourcing - -Lith's journal-first design naturally supports event sourcing patterns. - -=== Journal as Event Store - -[source] ----- -Lith Journal Entry = Event -─────────────────────────── -{ - "sequence": 1001, // Event sequence number - "timestamp": "2026-01-12T...", // Event time - "op_type": "DOC_INSERT", // Event type - "collection": "orders", // Aggregate type - "document_key": "ord_123", // Aggregate ID - "data": {...}, // Event payload - "provenance": { // Event metadata - "actor": "user:alice", - "rationale": "Order placed via checkout" - } -} ----- - -=== Event Projection - -.Projection Configuration -[source,toml] ----- -[projections.order_summary] -# Source: journal events -source_collection = "orders" - -# Target: materialized view -target_collection = "order_summaries" - -# Projection logic -[projections.order_summary.handlers] -DOC_INSERT = """ -UPSERT order_summaries -WHERE order_id = $event.data._key -SET - status = $event.data.status, - total = $event.data.total, - item_count = LEN($event.data.items), - last_updated = $event.timestamp -""" - -DOC_UPDATE = """ -UPDATE order_summaries -WHERE order_id = $event.document_key -SET - status = $event.data.status, - last_updated = $event.timestamp -""" ----- - -.Projection Implementation -[source,javascript] ----- -import { LithClient } from 'lith-client'; - -const lith = new LithClient('lith://localhost:8765'); - -// Subscribe to journal changes -const subscription = await lith.subscribeJournal({ - collection: 'orders', - startSequence: await getProjectionCheckpoint('order_summary') -}); - -for await (const event of subscription) { - switch (event.op_type) { - case 'DOC_INSERT': - await lith.query(` - INSERT INTO order_summaries { - "order_id": $1, - "status": $2, - "total": $3, - "item_count": $4, - "created_at": $5 - } - WITH PROVENANCE { - actor: "projection:order_summary", - rationale: "Materialized view update" - } - `, [ - event.data._key, - event.data.status, - event.data.total, - event.data.items.length, - event.timestamp - ]); - break; - - case 'DOC_UPDATE': - await lith.query(` - UPDATE order_summaries - WHERE order_id = $1 - SET status = $2, last_updated = $3 - WITH PROVENANCE { - actor: "projection:order_summary", - rationale: "Status update from order change" - } - `, [event.document_key, event.data.status, event.timestamp]); - break; - } - - await updateProjectionCheckpoint('order_summary', event.sequence); -} ----- - -=== Event Replay - -[source,bash] ----- -# Rebuild projection from scratch -lith projection rebuild \ - --name order_summary \ - --from-sequence 0 \ - --parallel 4 - -# Replay specific time range -lith projection rebuild \ - --name order_summary \ - --from-timestamp "2026-01-01T00:00:00Z" \ - --to-timestamp "2026-01-12T00:00:00Z" - -# Verify projection consistency -lith projection verify \ - --name order_summary \ - --sample-rate 0.1 ----- - -== Change Data Capture (CDC) - -=== CDC Architecture - -[source] ----- -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Lith │ │ CDC │ │ External │ -│ Journal │────▶│ Connector │────▶│ Systems │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ │ - │ ┌────┴────┐ - │ │Watermark│ - │ │ Storage │ - │ └─────────┘ - │ │ - └───────────────────┘ - High watermark stored - in Lith ----- - -=== Debezium-Compatible CDC - -.CDC Connector Configuration -[source,toml] ----- -[cdc] -enabled = true -connector_class = "io.debezium.connector.lith.LithConnector" - -# Source -[cdc.source] -lith_host = "lith://localhost:8765" -database = "production" -collections = ["evidence", "sources", "people"] - -# Sink (Kafka) -[cdc.sink] -bootstrap_servers = "kafka:9092" -topic_prefix = "lith.cdc" - -# Format -[cdc.format] -key_format = "json" -value_format = "json" -include_schema = true # Debezium envelope - -# Watermark tracking -[cdc.watermark] -storage = "lith" # Store watermarks in Lith -collection = "_cdc_watermarks" ----- - -.CDC Message Format (Debezium Envelope) -[source,json] ----- -{ - "schema": {...}, - "payload": { - "before": null, - "after": { - "_key": "ev_12345", - "title": "New evidence", - "score": 85 - }, - "source": { - "version": "0.0.2", - "connector": "lith", - "name": "production", - "ts_ms": 1704980400000, - "sequence": 100542, - "collection": "evidence", - "actor": "user:alice", - "rationale": "Evidence submission" - }, - "op": "c", // c=create, u=update, d=delete - "ts_ms": 1704980400000 - } -} ----- - -=== Webhook Publisher - -.Webhook Configuration -[source,toml] ----- -[webhooks] -enabled = true - -[[webhooks.endpoints]] -name = "slack-notifications" -url = "https://hooks.slack.com/services/xxx" -collections = ["evidence"] -operations = ["DOC_INSERT"] -filter = "score > 90" # Only high-score evidence -format = "slack" -retry_max = 3 -retry_delay_ms = 1000 - -[[webhooks.endpoints]] -name = "analytics-pipeline" -url = "https://analytics.example.com/ingest" -collections = ["*"] -operations = ["*"] -format = "json" -headers = { "Authorization" = "Bearer ${ANALYTICS_API_KEY}" } -batch_size = 100 -batch_delay_ms = 5000 ----- - -.Webhook Payload -[source,json] ----- -{ - "event_id": "evt_abc123", - "timestamp": "2026-01-12T10:30:00Z", - "source": "lith", - "type": "evidence.created", - "sequence": 100542, - "data": { - "_key": "ev_12345", - "title": "Critical evidence", - "score": 95 - }, - "provenance": { - "actor": "user:alice", - "rationale": "High-priority submission" - } -} ----- - -== API Gateway Integration - -=== GraphQL Federation - -.GraphQL Schema -[source,graphql] ----- -type Evidence @key(fields: "_key") { - _key: ID! - title: String! - content: String - source: String - score: Int! - createdAt: DateTime! - provenance: Provenance! - - # Relationships via edges - authors: [Person!]! @requires(fields: "_key") - citedSources: [Source!]! @requires(fields: "_key") -} - -type Provenance { - actor: String! - rationale: String! - timestamp: DateTime! -} - -type Query { - evidence(key: ID!): Evidence - searchEvidence(query: String!, limit: Int = 10): [Evidence!]! - evidenceByScore(minScore: Int!, limit: Int = 10): [Evidence!]! -} - -type Mutation { - createEvidence(input: EvidenceInput!, provenance: ProvenanceInput!): Evidence! - updateEvidence(key: ID!, input: EvidenceInput!, provenance: ProvenanceInput!): Evidence! -} - -input ProvenanceInput { - actor: String! - rationale: String! -} ----- - -.GraphQL Resolver -[source,javascript] ----- -import { LithClient } from 'lith-client'; - -const lith = new LithClient('lith://localhost:8765'); - -const resolvers = { - Query: { - evidence: async (_, { key }) => { - const result = await lith.query( - `SELECT * FROM evidence WHERE _key = $1`, - [key] - ); - return result[0]; - }, - - searchEvidence: async (_, { query, limit }) => { - return lith.query(` - SELECT * FROM evidence - WHERE SEARCH($1) - LIMIT $2 - `, [query, limit]); - } - }, - - Mutation: { - createEvidence: async (_, { input, provenance }) => { - return lith.insert('evidence', input, { - actor: provenance.actor, - rationale: provenance.rationale - }); - } - }, - - Evidence: { - authors: async (evidence) => { - return lith.query(` - TRAVERSE - START evidence WHERE _key = $1 - FOLLOW authored_by - RETURN people.* - `, [evidence._key]); - } - } -}; ----- - -=== REST API Gateway - -.Kong Gateway Configuration -[source,yaml] ----- -services: - - name: lith-api - url: http://lith:8765 - routes: - - name: lith-collections - paths: - - /api/v1/collections - methods: - - GET - - POST - plugins: - - name: rate-limiting - config: - minute: 100 - - name: jwt - - - name: lith-graphql - url: http://lith-graphql:4000 - routes: - - name: graphql - paths: - - /graphql - plugins: - - name: cors - - name: jwt ----- - -== AI/ML Pipelines - -=== Vector Embeddings - -.Vector Integration Configuration -[source,toml] ----- -[integrations.vector] -enabled = true - -# Embedding model -[integrations.vector.embedding] -provider = "openai" # openai, cohere, huggingface, local -model = "text-embedding-3-small" -dimensions = 1536 - -# Vector database -[integrations.vector.store] -type = "pinecone" # pinecone, weaviate, qdrant, chroma -index = "lith-evidence" - -# Sync settings -[integrations.vector.sync] -collections = ["evidence", "articles"] -fields_to_embed = ["title", "content"] -batch_size = 100 -checkpoint_collection = "_vector_checkpoints" ----- - -.Vector Sync Implementation -[source,python] ----- -import openai -from pinecone import Pinecone -from lith import LithClient - -openai_client = openai.OpenAI() -pc = Pinecone() -index = pc.Index("lith-evidence") -lith = LithClient("lith://localhost:8765") - -async def sync_vectors(): - checkpoint = await lith.query( - "SELECT sequence FROM _vector_checkpoints WHERE collection = 'evidence'" - ) - last_seq = checkpoint[0]['sequence'] if checkpoint else 0 - - entries = await lith.query(""" - SELECT * FROM _journal - WHERE sequence > $1 AND collection = 'evidence' - ORDER BY sequence LIMIT 100 - """, [last_seq]) - - for entry in entries: - if entry['op_type'] in ('DOC_INSERT', 'DOC_UPDATE'): - # Generate embedding - text = f"{entry['data']['title']} {entry['data'].get('content', '')}" - response = openai_client.embeddings.create( - model="text-embedding-3-small", - input=text - ) - vector = response.data[0].embedding - - # Upsert to Pinecone - index.upsert(vectors=[{ - "id": entry['document_key'], - "values": vector, - "metadata": { - "collection": "evidence", - "title": entry['data']['title'], - "score": entry['data'].get('score', 0), - "source": entry['data'].get('source', '') - } - }]) - - elif entry['op_type'] == 'DOC_DELETE': - index.delete(ids=[entry['document_key']]) - - if entries: - max_seq = max(e['sequence'] for e in entries) - await lith.query(""" - UPSERT _vector_checkpoints - WHERE collection = 'evidence' - SET sequence = $1 - WITH PROVENANCE { - actor: 'vector-sync', - rationale: 'Vector embedding checkpoint' - } - """, [max_seq]) ----- - -=== Semantic Search in GQL - -[source,gql] ----- --- Semantic search using vector similarity -SELECT * FROM evidence -WHERE SIMILAR_TO("documents about financial corruption in government", 0.8) - AND score > 70 -ORDER BY _similarity DESC -LIMIT 10 -WITH PROVENANCE { - actor: "investigator:jane", - rationale: "Semantic search for corruption investigation" -}; ----- - -=== LLM Integration - -.LLM Agent Configuration -[source,toml] ----- -[integrations.llm] -enabled = true - -[integrations.llm.agent] -# Agent identification in provenance -actor_prefix = "agent:" -model_tracking = true # Track model version in provenance - -# Rate limiting -requests_per_minute = 60 -tokens_per_minute = 100000 - -# Audit requirements -require_rationale = true -log_prompts = true -log_responses = true ----- - -.LLM Agent Usage -[source,javascript] ----- -import { LithClient } from 'lith-client'; -import OpenAI from 'openai'; - -const lith = new LithClient('lith://localhost:8765'); -const openai = new OpenAI(); - -async function classifyDocument(documentKey) { - // Retrieve document - const doc = await lith.query( - `SELECT * FROM evidence WHERE _key = $1`, - [documentKey] - ); - - // LLM classification - const response = await openai.chat.completions.create({ - model: "gpt-4-turbo", - messages: [ - { role: "system", content: "Classify the following document..." }, - { role: "user", content: doc.content } - ] - }); - - const classification = response.choices[0].message.content; - - // Update with full provenance - await lith.query(` - UPDATE evidence - WHERE _key = $1 - SET - category = $2, - ai_classification = $3 - WITH PROVENANCE { - actor: $4, - rationale: $5 - } - `, [ - documentKey, - classification.category, - classification, - `agent:document-classifier@model:gpt-4-turbo`, - `AI classification confidence=${classification.confidence}` - ]); -} ----- - -=== Human-in-the-Loop Patterns - -[source,gql] ----- --- Find AI-generated content pending review -SELECT * FROM evidence -WHERE _provenance.actor LIKE 'agent:%' - AND review_status = 'pending' -ORDER BY created_at; - --- Approve AI-generated content -UPDATE evidence -WHERE _key = 'ev_12345' -SET review_status = 'approved' -WITH PROVENANCE { - actor: "reviewer:jane", - rationale: "Verified AI classification is accurate" -}; - --- Audit AI agent activity -SELECT - _provenance.actor as agent, - COUNT(*) as actions, - AVG(confidence) as avg_confidence -FROM evidence -WHERE _provenance.actor LIKE 'agent:%' - AND created_at > NOW() - INTERVAL '7 days' -GROUP BY _provenance.actor; ----- - -== Real-Time Applications - -=== WebSocket Subscriptions - -.WebSocket Configuration -[source,toml] ----- -[realtime.websocket] -enabled = true -port = 8766 -path = "/ws" - -# Authentication -auth_required = true -auth_method = "jwt" - -# Rate limiting -max_connections_per_client = 10 -max_subscriptions_per_connection = 50 - -# Heartbeat -ping_interval_seconds = 30 -pong_timeout_seconds = 10 ----- - -.WebSocket Subscription Protocol -[source,javascript] ----- -// Client-side WebSocket subscription -const ws = new WebSocket('wss://lith.example.com/ws'); - -ws.onopen = () => { - // Authenticate - ws.send(JSON.stringify({ - type: 'auth', - token: 'jwt-token-here' - })); - - // Subscribe to collection changes - ws.send(JSON.stringify({ - type: 'subscribe', - id: 'sub_001', - collection: 'evidence', - filter: { score: { $gt: 80 } }, - operations: ['INSERT', 'UPDATE'] - })); -}; - -ws.onmessage = (event) => { - const message = JSON.parse(event.data); - - switch (message.type) { - case 'change': - console.log('Document changed:', message.data); - // { operation: 'INSERT', document: {...}, provenance: {...} } - break; - case 'error': - console.error('Subscription error:', message.error); - break; - } -}; ----- - -=== Server-Sent Events (SSE) - -.SSE Configuration -[source,toml] ----- -[realtime.sse] -enabled = true -path = "/events" - -# Connection settings -keep_alive_seconds = 30 -max_connections = 1000 ----- - -.SSE Subscription -[source,javascript] ----- -// Client-side SSE subscription -const eventSource = new EventSource( - 'https://lith.example.com/events?' + - 'collection=evidence&' + - 'filter=score>80&' + - 'token=jwt-token' -); - -eventSource.onmessage = (event) => { - const change = JSON.parse(event.data); - console.log('Evidence changed:', change); -}; - -eventSource.onerror = (error) => { - console.error('SSE error:', error); - eventSource.close(); -}; ----- - -== Authentication Provider Integration - -=== OIDC/OAuth2 Integration - -.OIDC Configuration -[source,toml] ----- -[auth.oidc] -enabled = true -issuer = "https://auth.example.com" -client_id = "${OIDC_CLIENT_ID}" -client_secret = "${OIDC_CLIENT_SECRET}" - -# Token validation -audience = "lith-api" -algorithms = ["RS256"] - -# User mapping -[auth.oidc.claims] -actor = "sub" # JWT claim for actor identity -email = "email" -name = "name" -roles = "roles" - -# Role to permission mapping -[auth.oidc.roles] -admin = ["*"] -editor = ["read", "write"] -viewer = ["read"] ----- - -See link:SECURITY-AUTH.adoc[Security & Authentication] for complete authentication configuration. - -== Monitoring Integration - -=== Prometheus Metrics Export - -See link:OBSERVABILITY.adoc[Observability Guide] for complete metrics configuration. - -=== Datadog Integration - -.Datadog Configuration -[source,toml] ----- -[integrations.datadog] -enabled = true -api_key = "${DATADOG_API_KEY}" -site = "datadoghq.com" - -[integrations.datadog.metrics] -prefix = "lith" -tags = ["env:production", "service:lith"] - -[integrations.datadog.logs] -enabled = true -source = "lith" -service = "lith" - -[integrations.datadog.apm] -enabled = true -service = "lith" -env = "production" ----- - -== See Also - -* link:API-REFERENCE.adoc[API Reference] - Complete API documentation -* link:SECURITY-AUTH.adoc[Security & Authentication] - Authentication configuration -* link:OBSERVABILITY.adoc[Observability Guide] - Monitoring and metrics -* link:DEPLOYMENT.adoc[Deployment Guide] - Production deployment -* link:../ARCHITECTURE.adoc[Architecture] - Technical design -* link:../spec/journal.adoc[Journal Specification] - Journal format details diff --git a/lithoglyph/docs/INTEGRATION-PLAN-LITHOGLYPH.md b/lithoglyph/docs/INTEGRATION-PLAN-LITHOGLYPH.md deleted file mode 100644 index 5b0a1aa1..00000000 --- a/lithoglyph/docs/INTEGRATION-PLAN-LITHOGLYPH.md +++ /dev/null @@ -1,111 +0,0 @@ -# Lithoglyph Integration Plan — Pipeline Role -# -# SPDX-License-Identifier: CC-BY-SA-4.0 -# Author: Jonathan D.A. Jewell -# Created: 2026-03-13 -# -# Extracted from the master integration plan: bofig/docs/INTEGRATION-PLAN.md - -## Lithoglyph's Role in the Pipeline - -Lithoglyph is the **provenance layer**. It stores all evidence with full audit -trail, reversibility, and PROMPT scoring. It sits between Docudactyl (extraction) -and Bofig (evidence graph navigation). - -``` -Raw Documents (200K+ files) - → Docudactyl (HPC extraction: OCR, NER, metadata, classification) - → Lithoglyph (audit-grade storage: provenance, reversibility, PROMPT) - → Bofig (evidence graph: claims, relationships, navigation) -``` - -## Lithoglyph Tasks (from Integration Plan) - -| # | Task | Priority | Effort | Notes | -|---|------|----------|--------|-------| -| L1 | Zig 0.15.2 HTTP API migration (83 call sites) | Critical | Medium | **COMPLETE** — Reader/Writer pattern applied | -| L2 | Lith → Lithoglyph rename (Google trademark) | Critical | Small | **COMPLETE** — formdb-http/ → lith-http/ | -| L3 | Evidence collection schema for bofig | High | Small | **COMPLETE** — 5 collections + 5 GQL test vectors | -| L4 | Financial transaction collection | High | Small | source, destination, amount, date, instrument, intermediary | -| L5 | Entity collection + co-reference resolution | High | Medium | Person/org/location entities with alias tracking | -| L6 | Ingest bridge: Docudactyl Cap'n Proto → GQL INSERT | High | Medium | Batch import with auto-PROMPT scoring from extraction metadata | -| L7 | Temporal credibility model | Medium | Medium | Source reputation updates over time (retractions, discrediting) | -| L8 | Cross-investigation linking | Medium | Small | Shared evidence collections, automatic surfacing | -| L9 | ControlPlane clustering (Elixir/OTP) | Low | Large | Multi-node Lithoglyph for scale | - -## Key Guarantees Lithoglyph Provides - -- Every mutation has actor + rationale (accountability) -- Reversible operations (retractions with explanation) -- Time-travel queries ("what did we know on date X?") -- PROMPT scores as first-class citizens -- Constraints-as-ethics (invalid evidence relationships rejected with explanation) -- Dependent-type proofs (GQL-DT) for score bounds - -## Integration Points Involving Lithoglyph - -### Integration 1: Docudactyl → Lithoglyph (D2 + L6) - -``` -Docudactyl Cap'n Proto output - → Adapter (D2) serializes to GQL INSERT statements - → Lithoglyph ingest bridge (L6) batch-imports with: - - Auto-PROMPT scoring from extraction confidence - - SHA-256 dedup against existing evidence - - Actor="docudactyl-pipeline", Rationale="Batch extraction run {id}" - - Provenance: source file path, extraction timestamp, OCR confidence -``` - -### Integration 2: Lithoglyph → Bofig (L3 + B5) - -``` -Lithoglyph evidence/entity/transaction collections - → Bofig GenServer (B5) queries Lithoglyph via GQL - → Maps to ArangoDB graph (Phase 2) or direct Lithoglyph queries (Phase 3) - → PROMPT scores flow from Lithoglyph → bofig UI - → Provenance metadata available on hover/click in UI -``` - -### Integration 3: Entity Resolution Loop (D3/D4/D5 → L5 → B1) - -``` -Docudactyl NER extracts raw entities - → Lithoglyph stores with alias tracking (L5) - → Bofig entity resolution (B1) merges aliases - → Merge decision logged in Lithoglyph journal with rationale - → Reversible if co-reference was incorrect -``` - -### Integration 4: Financial Flow Analysis (D4 → L4 → B2) - -``` -Docudactyl extracts transactions from bank records (D4) - → Lithoglyph financial_transactions collection (L4) - → Bofig GraphQL: transactionChain(entityId, depth) (B2) -``` - -### Integration 5: Temporal Reconstruction (D3 → L7 → B3) - -``` -Docudactyl extracts dates from all documents - → Lithoglyph stores with temporal metadata - → Bofig timeline view (B3): - - What happened when / what was known when - - Source credibility at each point in time (L7) -``` - -## Phase Assignment - -Lithoglyph work spans **Phase A (Foundation)** through **Phase D (Scale & Trust)**: - -- **Phase A (Weeks 1-4):** L1 (Zig migration), L2 (rename), L3 (evidence schema) -- **Phase B (Weeks 5-8):** L4 (financial collection), L5 (entity collection), L6 (ingest bridge) -- **Phase C (Weeks 9-14):** (no Lithoglyph-primary tasks) -- **Phase D (Weeks 15-20):** L7 (temporal credibility), L8 (cross-investigation linking) - -## Cross-References - -- **Master plan:** `bofig/docs/INTEGRATION-PLAN.md` -- **Epstein worked example:** `bofig/docs/EPSTEIN-FILES-WORK-PATHWAY.md` -- **Epstein Lithoglyph phases:** `docs/EPSTEIN-INGEST-TESTS.md` (this repo) -- **GQL dependent types spec:** `spec/gql-dependent-types.md` (this repo) diff --git a/lithoglyph/docs/MIGRATION-FROM-RDBMS.adoc b/lithoglyph/docs/MIGRATION-FROM-RDBMS.adoc deleted file mode 100644 index ce416a2f..00000000 --- a/lithoglyph/docs/MIGRATION-FROM-RDBMS.adoc +++ /dev/null @@ -1,1468 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Migrating to Lith from RDBMS -:toc: left -:toclevels: 4 -:icons: font -:source-highlighter: rouge -:sectanchors: -:sectlinks: - -[.lead] -Comprehensive guide for migrating from traditional relational databases (PostgreSQL, MySQL, SQLite) to Lith, including schema translation, data migration, and provenance strategies. - -[NOTE] -==== -**Status: Documentation Complete, Tooling Planned** - -This document provides complete migration guidance. Migration tooling (`lith-migrate` CLI) is planned for Milestone M12. Currently, migrations use the documented manual processes and scripts. -==== - -== Philosophy: From SQL to Narrative - -Before beginning a migration, understand that Lith is not a drop-in replacement for traditional RDBMS. It represents a fundamental shift in how data is treated: - -[cols="1,2,2"] -|=== -| Aspect | Traditional RDBMS | Lith - -| **Primary Concern** -| Performance, ACID transactions -| Auditability, provenance, reversibility - -| **Data Model** -| Tables with rows -| Documents and edges (graph-document hybrid) - -| **History** -| Optional (audit tables, triggers) -| Built-in (every operation journaled) - -| **Deletions** -| Permanent (unless soft-delete) -| Reversible (tombstones with inverse) - -| **Mutations** -| Who/when via triggers -| Who/why/when required (provenance) - -| **Schema Evolution** -| Manual migrations -| Self-normalizing with proofs - -| **Query Philosophy** -| Declarative (what) -| Narrative (why + what) -|=== - -=== When Lith is the Right Choice - -Lith excels for: - -* **Investigative Journalism**: Complete chain of custody for evidence -* **Compliance/Governance**: Built-in audit trails for regulators -* **Agentic AI Systems**: Explain and reverse AI decisions -* **Long-Term Archives**: Decades-scale data with provenance -* **Research Data**: Reproducible data transformations -* **Legal Discovery**: Immutable evidence trails - -=== When to Stay with RDBMS - -Lith is NOT suitable for: - -* **High-Throughput OLTP**: >10,000 TPS requirements -* **Analytics/OLAP**: Columnar analytics, data warehousing -* **Drop-In Replacement**: Apps expecting SQL compatibility -* **Performance-Critical**: Sub-millisecond query requirements -* **Ephemeral Data**: Cache, sessions, temp data - -== Migration Assessment - -=== Readiness Checklist - -Before migrating, assess your current system: - -[cols="1,3,1"] -|=== -| Item | Description | Ready? - -| **Data Volume** -| Lith handles datasets up to ~100GB efficiently. Larger datasets require tiered storage. -| ☐ - -| **Transaction Rate** -| Maximum ~1,000 TPS per node. Higher requires sharding (🚧 planned). -| ☐ - -| **Query Complexity** -| Complex JOINs become edge traversals. Assess query patterns. -| ☐ - -| **Audit Requirements** -| Lith provides superior audit trails—verify this is needed. -| ☐ - -| **Schema Stability** -| Self-normalizing handles evolution, but frequent changes add overhead. -| ☐ - -| **Application Changes** -| Applications need modification for GQL and provenance. -| ☐ - -| **Team Training** -| Developers need GQL and provenance concept training. -| ☐ -|=== - -=== Data Inventory - -Catalog your existing database: - -[source,sql] ----- --- PostgreSQL: List all tables and row counts -SELECT - schemaname, - tablename, - n_tup_ins - n_tup_del as row_count_estimate -FROM pg_stat_user_tables -ORDER BY row_count_estimate DESC; - --- List foreign key relationships (become edges) -SELECT - tc.table_name as from_table, - kcu.column_name as from_column, - ccu.table_name as to_table, - ccu.column_name as to_column -FROM information_schema.table_constraints tc -JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name -JOIN information_schema.constraint_column_usage ccu - ON ccu.constraint_name = tc.constraint_name -WHERE tc.constraint_type = 'FOREIGN KEY'; ----- - -[source,sql] ----- --- MySQL: List all tables and row counts -SELECT - table_schema, - table_name, - table_rows -FROM information_schema.tables -WHERE table_schema = 'your_database' -ORDER BY table_rows DESC; - --- List foreign key relationships -SELECT - table_name as from_table, - column_name as from_column, - referenced_table_name as to_table, - referenced_column_name as to_column -FROM information_schema.key_column_usage -WHERE referenced_table_name IS NOT NULL - AND table_schema = 'your_database'; ----- - -=== Migration Complexity Score - -Calculate migration complexity: - -[source] ----- -Score = (Tables × 1) + (Foreign Keys × 2) + (Stored Procedures × 5) - + (Triggers × 3) + (Views × 2) - -Complexity Levels: - 0-20: Simple (1-2 weeks) - 21-50: Moderate (2-4 weeks) - 51-100: Complex (1-2 months) - 100+: Enterprise (3+ months, phased approach) ----- - -== Schema Translation - -=== Type Mapping Reference - -[cols="1,1,2"] -|=== -| SQL Type | GQL Type | Notes - -3+h| **Numeric Types** - -| `SMALLINT`, `INTEGER`, `BIGINT` -| `INTEGER` -| Arbitrary precision in Lith - -| `DECIMAL(p,s)`, `NUMERIC(p,s)` -| `DECIMAL(p,s)` 🚧 -| Planned. Use `FLOAT` or `STRING` for now - -| `REAL`, `FLOAT`, `DOUBLE` -| `FLOAT` -| IEEE 754 double precision - -| `SERIAL`, `BIGSERIAL` -| (auto-generated) -| Lith uses UUIDs by default - -3+h| **String Types** - -| `CHAR(n)`, `VARCHAR(n)` -| `STRING` -| No length limits in Lith - -| `TEXT` -| `STRING` -| Same as VARCHAR - -| `BYTEA`, `BLOB` -| `BINARY` -| Base64 encoded in JSON - -3+h| **Date/Time Types** - -| `DATE` -| `DATE` -| ISO 8601 date - -| `TIME` -| `TIME` -| ISO 8601 time - -| `TIMESTAMP`, `DATETIME` -| `TIMESTAMP` -| ISO 8601 with timezone - -| `INTERVAL` -| `DURATION` 🚧 -| Planned. Use ISO 8601 duration string - -3+h| **Boolean & JSON** - -| `BOOLEAN` -| `BOOLEAN` -| true/false - -| `JSON`, `JSONB` -| `JSON` -| Native JSON support - -3+h| **Special Types** - -| `UUID` -| `UUID` -| Native UUID support - -| `ARRAY[]` -| JSON array -| Use `["a", "b", "c"]` syntax - -| `ENUM(...)` -| `STRING` + constraint -| Add CHECK constraint for valid values - -| `POINT`, `GEOMETRY` -| `GEO` 🚧 -| Planned. Use GeoJSON for now - -3+h| **Lith-Specific Types** - -| (no equivalent) -| `PROMPT_SCORE` -| 0-100 evidence quality rating - -| (no equivalent) -| `PROVENANCE` -| Actor + rationale (auto-managed) - -| (no equivalent) -| `DOCUMENT_REF` -| Reference to another document -|=== - -=== Constraint Translation - -[cols="1,2,2"] -|=== -| SQL Constraint | GQL Equivalent | Example - -| `NOT NULL` -| `NOT NULL` -| `title STRING NOT NULL` - -| `UNIQUE` -| `UNIQUE` -| `email STRING UNIQUE` - -| `PRIMARY KEY` -| (auto-generated `_key`) -| Documents have automatic UUIDs - -| `CHECK (expr)` -| `CHECK (expr)` -| `CHECK (score >= 0 AND score <= 100)` - -| `DEFAULT value` -| `DEFAULT value` -| `status STRING DEFAULT 'draft'` - -| `FOREIGN KEY` -| Edge collection -| See relationship migration section -|=== - -=== Schema Migration Examples - -==== Simple Table Migration - -.PostgreSQL Source -[source,sql] ----- -CREATE TABLE articles ( - id SERIAL PRIMARY KEY, - title VARCHAR(255) NOT NULL, - content TEXT, - author_id INTEGER REFERENCES users(id), - published_at TIMESTAMP, - views INTEGER DEFAULT 0, - status VARCHAR(20) CHECK (status IN ('draft', 'published', 'archived')), - metadata JSONB -); ----- - -.Lith Target -[source,gql] ----- -CREATE COLLECTION articles ( - title STRING NOT NULL, - content STRING, - published_at TIMESTAMP, - views INTEGER DEFAULT 0, - status STRING DEFAULT 'draft' - CHECK (status IN ('draft', 'published', 'archived')), - metadata JSON -) WITH DESCRIPTION 'News articles with publication workflow'; - --- Foreign key becomes an edge collection -CREATE EDGE COLLECTION authored_by -FROM articles TO users -WITH PROPERTIES ( - role STRING DEFAULT 'author' -); ----- - -==== Complex Table with Multiple Relationships - -.PostgreSQL Source -[source,sql] ----- -CREATE TABLE orders ( - id SERIAL PRIMARY KEY, - order_number VARCHAR(50) UNIQUE NOT NULL, - customer_id INTEGER NOT NULL REFERENCES customers(id), - shipping_address_id INTEGER REFERENCES addresses(id), - billing_address_id INTEGER REFERENCES addresses(id), - status VARCHAR(30) NOT NULL DEFAULT 'pending', - total_amount DECIMAL(10,2) NOT NULL, - tax_amount DECIMAL(10,2) DEFAULT 0, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP, - CONSTRAINT valid_status CHECK ( - status IN ('pending', 'confirmed', 'shipped', 'delivered', 'cancelled') - ), - CONSTRAINT positive_amounts CHECK ( - total_amount >= 0 AND tax_amount >= 0 - ) -); - -CREATE TABLE order_items ( - id SERIAL PRIMARY KEY, - order_id INTEGER NOT NULL REFERENCES orders(id) ON DELETE CASCADE, - product_id INTEGER NOT NULL REFERENCES products(id), - quantity INTEGER NOT NULL CHECK (quantity > 0), - unit_price DECIMAL(10,2) NOT NULL, - discount_percent DECIMAL(5,2) DEFAULT 0 -); ----- - -.Lith Target -[source,gql] ----- --- Main orders collection -CREATE COLLECTION orders ( - order_number STRING UNIQUE NOT NULL, - status STRING NOT NULL DEFAULT 'pending' - CHECK (status IN ('pending', 'confirmed', 'shipped', 'delivered', 'cancelled')), - total_amount FLOAT NOT NULL CHECK (total_amount >= 0), - tax_amount FLOAT DEFAULT 0 CHECK (tax_amount >= 0), - notes STRING, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP -) WITH DESCRIPTION 'Customer orders with status tracking'; - --- Customer relationship -CREATE EDGE COLLECTION placed_by -FROM orders TO customers -WITH PROPERTIES ( - placed_at TIMESTAMP DEFAULT NOW() -); - --- Address relationships (two edge types for different purposes) -CREATE EDGE COLLECTION ships_to -FROM orders TO addresses -WITH PROPERTIES ( - address_type STRING DEFAULT 'shipping' -); - -CREATE EDGE COLLECTION bills_to -FROM orders TO addresses -WITH PROPERTIES ( - address_type STRING DEFAULT 'billing' -); - --- Order items as separate collection with edges -CREATE COLLECTION order_items ( - quantity INTEGER NOT NULL CHECK (quantity > 0), - unit_price FLOAT NOT NULL, - discount_percent FLOAT DEFAULT 0 -) WITH DESCRIPTION 'Individual items within an order'; - --- Relationships for order items -CREATE EDGE COLLECTION contains_item -FROM orders TO order_items -WITH PROPERTIES ( - line_number INTEGER -); - -CREATE EDGE COLLECTION item_is_product -FROM order_items TO products; ----- - -== Database-Specific Migration - -=== PostgreSQL Migration - -==== Export Data - -[source,bash] ----- -#!/bin/bash -# export-postgres.sh - Export PostgreSQL database for Lith migration - -DATABASE="your_database" -OUTPUT_DIR="./lith-export" -mkdir -p "$OUTPUT_DIR" - -# Export each table as JSON (preferred format) -for table in $(psql -d "$DATABASE" -t -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public'"); do - echo "Exporting $table..." - psql -d "$DATABASE" -c "\COPY (SELECT row_to_json(t) FROM (SELECT * FROM $table) t) TO '$OUTPUT_DIR/${table}.jsonl'" -done - -# Export schema information -pg_dump --schema-only "$DATABASE" > "$OUTPUT_DIR/schema.sql" - -# Export foreign key relationships for edge creation -psql -d "$DATABASE" -t -c " - SELECT json_build_object( - 'from_table', tc.table_name, - 'from_column', kcu.column_name, - 'to_table', ccu.table_name, - 'to_column', ccu.column_name, - 'constraint_name', tc.constraint_name - ) - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - JOIN information_schema.constraint_column_usage ccu - ON ccu.constraint_name = tc.constraint_name - WHERE tc.constraint_type = 'FOREIGN KEY' -" > "$OUTPUT_DIR/relationships.jsonl" - -echo "Export complete. Files in $OUTPUT_DIR" ----- - -==== Import to Lith - -[source,gql] ----- --- Create collections based on exported schema --- (Run schema translation first) - --- Import documents with default provenance for migration -INSERT INTO articles -IMPORT FROM 'articles.jsonl' -WITH PROVENANCE { - actor: "migration-script", - rationale: "Bulk import from PostgreSQL migration on 2026-01-12" -}; - --- Verify import -SELECT COUNT(*) FROM articles; -INTROSPECT COLLECTION articles; ----- - -==== Handle PostgreSQL-Specific Features - -[cols="1,2"] -|=== -| PostgreSQL Feature | Lith Approach - -| **Schemas (namespaces)** -| Prefix collection names: `public_users` or use separate databases - -| **Sequences** -| Use UUID `_key` (auto-generated) or custom ID generation - -| **Materialized Views** -| Create read-through collections (🚧 planned) - -| **Stored Procedures** -| Move logic to application layer - -| **Triggers** -| Use journal subscriptions (🚧 planned) - -| **Full-Text Search** -| Integration with Meilisearch/Typesense (🚧 planned) - -| **PostGIS Geometry** -| Use GeoJSON in JSON field, native GEO type planned - -| **Arrays** -| Use JSON arrays - -| **JSONB operators** -| Native JSON support with similar operators - -| **CTEs (WITH clause)** -| Chain queries or use TRAVERSE for graph queries -|=== - -=== MySQL Migration - -==== Export Data - -[source,bash] ----- -#!/bin/bash -# export-mysql.sh - Export MySQL database for Lith migration - -DATABASE="your_database" -OUTPUT_DIR="./lith-export" -mkdir -p "$OUTPUT_DIR" - -# Export each table as JSON -for table in $(mysql -N -e "SHOW TABLES" "$DATABASE"); do - echo "Exporting $table..." - mysql -N -e "SELECT JSON_OBJECT($(mysql -N -e "SELECT GROUP_CONCAT(CONCAT(\"'\", COLUMN_NAME, \"', \", COLUMN_NAME)) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='$DATABASE' AND TABLE_NAME='$table'")) FROM $table" "$DATABASE" > "$OUTPUT_DIR/${table}.jsonl" -done - -# Export schema -mysqldump --no-data "$DATABASE" > "$OUTPUT_DIR/schema.sql" - -# Export foreign keys -mysql -N -e " - SELECT JSON_OBJECT( - 'from_table', TABLE_NAME, - 'from_column', COLUMN_NAME, - 'to_table', REFERENCED_TABLE_NAME, - 'to_column', REFERENCED_COLUMN_NAME, - 'constraint_name', CONSTRAINT_NAME - ) - FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE - WHERE REFERENCED_TABLE_NAME IS NOT NULL - AND TABLE_SCHEMA = '$DATABASE' -" "$DATABASE" > "$OUTPUT_DIR/relationships.jsonl" - -echo "Export complete." ----- - -==== MySQL-Specific Considerations - -[cols="1,2"] -|=== -| MySQL Feature | Lith Approach - -| **AUTO_INCREMENT** -| Use UUID `_key` (auto-generated) - -| **ENUM type** -| STRING with CHECK constraint - -| **SET type** -| JSON array with validation - -| **Spatial types** -| GeoJSON in JSON field - -| **FULLTEXT indexes** -| External search integration - -| **Stored Procedures** -| Application layer logic - -| **Events (scheduled)** -| External scheduler (cron, systemd timers) - -| **Partitioning** -| Collection sharding (🚧 planned) -|=== - -=== SQLite Migration - -==== Export Data - -[source,bash] ----- -#!/bin/bash -# export-sqlite.sh - Export SQLite database for Lith migration - -DATABASE="your_database.db" -OUTPUT_DIR="./lith-export" -mkdir -p "$OUTPUT_DIR" - -# Get list of tables -tables=$(sqlite3 "$DATABASE" "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") - -# Export each table as JSON -for table in $tables; do - echo "Exporting $table..." - sqlite3 -json "$DATABASE" "SELECT * FROM $table" > "$OUTPUT_DIR/${table}.json" -done - -# Export schema -sqlite3 "$DATABASE" ".schema" > "$OUTPUT_DIR/schema.sql" - -echo "Export complete." ----- - -==== SQLite-Specific Considerations - -[cols="1,2"] -|=== -| SQLite Feature | Lith Approach - -| **ROWID/INTEGER PRIMARY KEY** -| Use UUID `_key` - -| **Dynamic typing** -| Explicit type declarations in GQL - -| **ATTACH DATABASE** -| Separate Lith databases - -| **Virtual tables (FTS5)** -| External search integration - -| **JSON1 extension** -| Native JSON support - -| **Window functions** -| Query decomposition or application logic -|=== - -== Relationship Migration - -Foreign keys in RDBMS become edge collections in Lith. This is one of the most significant changes in migration. - -=== Foreign Key to Edge Mapping - -.RDBMS Foreign Key -[source,sql] ----- --- articles.author_id → users.id -ALTER TABLE articles -ADD CONSTRAINT fk_author -FOREIGN KEY (author_id) REFERENCES users(id); ----- - -.Lith Edge Collection -[source,gql] ----- --- Remove author_id from articles collection --- Create edge collection instead -CREATE EDGE COLLECTION written_by -FROM articles TO users -WITH PROPERTIES ( - role STRING DEFAULT 'author' -); ----- - -=== Migration Script for Relationships - -[source,bash] ----- -#!/bin/bash -# migrate-relationships.sh - Convert foreign keys to edges - -# For each relationship in relationships.jsonl: -while IFS= read -r line; do - from_table=$(echo "$line" | jq -r '.from_table') - from_column=$(echo "$line" | jq -r '.from_column') - to_table=$(echo "$line" | jq -r '.to_table') - - # Generate edge collection name - edge_name="${from_table}_to_${to_table}" - - echo "Creating edge collection: $edge_name" - - # Generate GQL for edge creation - cat << EOF -CREATE EDGE COLLECTION $edge_name -FROM $from_table TO $to_table; - --- Migrate existing relationships -INSERT INTO $edge_name -SELECT - a._key as _from, - (SELECT _key FROM $to_table WHERE old_id = a.$from_column) as _to -FROM $from_table a -WHERE a.$from_column IS NOT NULL -WITH PROVENANCE { - actor: "migration-script", - rationale: "Converting foreign key $from_column to edge" -}; -EOF - -done < relationships.jsonl ----- - -=== Many-to-Many Relationships - -.RDBMS Junction Table -[source,sql] ----- -CREATE TABLE article_tags ( - article_id INTEGER REFERENCES articles(id), - tag_id INTEGER REFERENCES tags(id), - added_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (article_id, tag_id) -); ----- - -.Lith Edge Collection -[source,gql] ----- --- Junction table becomes edge collection with properties -CREATE EDGE COLLECTION tagged_with -FROM articles TO tags -WITH PROPERTIES ( - added_at TIMESTAMP DEFAULT NOW(), - added_by STRING -); - --- Migrate existing tag relationships -INSERT INTO tagged_with -SELECT - (SELECT _key FROM articles WHERE old_id = jt.article_id) as _from, - (SELECT _key FROM tags WHERE old_id = jt.tag_id) as _to, - jt.added_at as added_at -FROM article_tags_import jt -WITH PROVENANCE { - actor: "migration-script", - rationale: "Converting article_tags junction table" -}; ----- - -=== Self-Referencing Relationships - -.RDBMS Self-Reference -[source,sql] ----- -CREATE TABLE employees ( - id SERIAL PRIMARY KEY, - name VARCHAR(100), - manager_id INTEGER REFERENCES employees(id) -); ----- - -.Lith Self-Referencing Edge -[source,gql] ----- -CREATE COLLECTION employees ( - name STRING NOT NULL -); - -CREATE EDGE COLLECTION reports_to -FROM employees TO employees -WITH PROPERTIES ( - relationship STRING DEFAULT 'direct_report' -); - --- Query the hierarchy -TRAVERSE - START employees WHERE name = 'Jane' - FOLLOW reports_to DEPTH 3 - DIRECTION INBOUND - RETURN employees.name, DEPTH; ----- - -== Query Translation - -=== Basic Queries - -[cols="1,1"] -|=== -| SQL | GQL - -| `SELECT * FROM users` -| `SELECT * FROM users` - -| `SELECT name, email FROM users` -| `SELECT name, email FROM users` - -| `SELECT * FROM users WHERE active = true` -| `SELECT * FROM users WHERE active = true` - -| `SELECT * FROM users ORDER BY name` -| `SELECT * FROM users ORDER BY name` - -| `SELECT * FROM users LIMIT 10` -| `SELECT * FROM users LIMIT 10` - -| `SELECT * FROM users LIMIT 10 OFFSET 20` -| `SELECT * FROM users OFFSET 20 LIMIT 10` - -| `SELECT COUNT(*) FROM users` -| `SELECT COUNT(*) FROM users` - -| `SELECT DISTINCT status FROM orders` -| `SELECT DISTINCT status FROM orders` -|=== - -=== Aggregations - -[cols="1,1"] -|=== -| SQL | GQL - -| `SELECT COUNT(*), status FROM orders GROUP BY status` -| `SELECT COUNT(*), status FROM orders GROUP BY status` - -| `SELECT SUM(amount) FROM orders` -| `SELECT SUM(amount) FROM orders` - -| `SELECT AVG(score) FROM reviews WHERE product_id = 123` -| `SELECT AVG(score) FROM reviews WHERE product_id = '123'` - -| `SELECT MAX(created_at) FROM logs` -| `SELECT MAX(created_at) FROM logs` -|=== - -=== JOINs → TRAVERSE - -The biggest difference: SQL JOINs become graph traversals. - -.SQL JOIN -[source,sql] ----- -SELECT - a.title, - u.name as author_name -FROM articles a -JOIN users u ON a.author_id = u.id -WHERE a.status = 'published'; ----- - -.GQL TRAVERSE -[source,gql] ----- --- First, ensure edge collection exists --- Then traverse the relationship -TRAVERSE - START articles WHERE status = 'published' - FOLLOW written_by - RETURN articles.title, users.name as author_name; ----- - -.SQL Multiple JOINs -[source,sql] ----- -SELECT - o.order_number, - c.name as customer_name, - p.name as product_name, - oi.quantity -FROM orders o -JOIN customers c ON o.customer_id = c.id -JOIN order_items oi ON oi.order_id = o.id -JOIN products p ON oi.product_id = p.id -WHERE o.status = 'shipped'; ----- - -.GQL Multiple Traversals -[source,gql] ----- -TRAVERSE - START orders WHERE status = 'shipped' - FOLLOW placed_by, contains_item - FOLLOW item_is_product FROM order_items - RETURN - orders.order_number, - customers.name as customer_name, - products.name as product_name, - order_items.quantity; ----- - -=== Subqueries - -.SQL Subquery -[source,sql] ----- -SELECT * FROM users -WHERE id IN ( - SELECT DISTINCT author_id - FROM articles - WHERE status = 'published' -); ----- - -.GQL (using edge traversal) -[source,gql] ----- --- Reverse traversal: find users who have published articles -TRAVERSE - START articles WHERE status = 'published' - FOLLOW written_by - DIRECTION OUTBOUND - RETURN DISTINCT users.*; ----- - -=== INSERT Statements - -[cols="1,1"] -|=== -| SQL | GQL - -a| -[source,sql] ----- -INSERT INTO users (name, email) -VALUES ('John', 'john@example.com'); ----- -a| -[source,gql] ----- -INSERT INTO users { - "name": "John", - "email": "john@example.com" -} -WITH PROVENANCE { - actor: "api-user-123", - rationale: "User registration" -}; ----- -|=== - -=== UPDATE Statements - -[cols="1,1"] -|=== -| SQL | GQL - -a| -[source,sql] ----- -UPDATE users -SET email = 'new@example.com' -WHERE id = 123; ----- -a| -[source,gql] ----- -UPDATE users -WHERE _key = '123' -SET email = 'new@example.com' -WITH PROVENANCE { - actor: "admin-456", - rationale: "Email change request #789" -}; ----- -|=== - -=== DELETE Statements - -[cols="1,1"] -|=== -| SQL | GQL - -a| -[source,sql] ----- -DELETE FROM users -WHERE id = 123; ----- -a| -[source,gql] ----- -DELETE FROM users -WHERE _key = '123' -WITH PROVENANCE { - actor: "admin-456", - rationale: "GDPR deletion request #101" -}; ----- -|=== - -Note: Lith DELETE creates a tombstone with inverse operation, allowing UNDO. - -== Provenance Strategy - -Provenance is mandatory for all mutations in Lith. Plan how to capture actor and rationale. - -=== Application Architecture Changes - -.Before (RDBMS) -[source,javascript] ----- -// Simple insert - no audit context -async function createArticle(title, content, authorId) { - await db.query( - 'INSERT INTO articles (title, content, author_id) VALUES ($1, $2, $3)', - [title, content, authorId] - ); -} ----- - -.After (Lith) -[source,javascript] ----- -// Insert with provenance context -async function createArticle(title, content, authorId, context) { - await lith.query(` - INSERT INTO articles { - "title": $1, - "content": $2 - } - WITH PROVENANCE { - actor: $3, - rationale: $4 - } - `, [title, content, context.userId, context.reason]); - - // Create edge to author - await lith.query(` - CREATE EDGE written_by - FROM articles/_last TO users/$1 - WITH PROVENANCE { - actor: $2, - rationale: "Article authorship" - } - `, [authorId, context.userId]); -} ----- - -=== Provenance Context Middleware - -.Express.js Middleware Example -[source,javascript] ----- -// middleware/provenance.js -function provenanceMiddleware(req, res, next) { - // Extract user from JWT or session - const userId = req.user?.id || 'anonymous'; - const sessionId = req.sessionID; - - // Attach provenance context to request - req.provenance = { - actor: `user:${userId}@session:${sessionId}`, - // Rationale comes from request or defaults - rationale: req.body._rationale || req.query._rationale || - `${req.method} ${req.path} from ${req.ip}` - }; - - next(); -} - -// Usage in route -app.post('/articles', provenanceMiddleware, async (req, res) => { - const { title, content } = req.body; - - await lith.insert('articles', { title, content }, { - actor: req.provenance.actor, - rationale: req.body.rationale || 'Created via API' - }); -}); ----- - -=== Bulk Migration Provenance - -For migrated data, use migration-specific provenance: - -[source,gql] ----- --- Set migration context for bulk imports -SET PROVENANCE DEFAULTS { - actor: "migration-script-v1.0", - rationale: "Initial data migration from PostgreSQL (production backup 2026-01-10)" -}; - --- Bulk imports use default provenance -INSERT INTO articles IMPORT FROM 'articles.jsonl'; -INSERT INTO users IMPORT FROM 'users.jsonl'; - --- Clear defaults when done -CLEAR PROVENANCE DEFAULTS; ----- - -=== Provenance Patterns by Use Case - -[cols="1,2,2"] -|=== -| Scenario | Actor Format | Rationale Example - -| User action -| `user:123@session:abc` -| "Updated profile email" - -| API integration -| `service:payment-processor` -| "Payment confirmed #PAY-456" - -| Scheduled job -| `job:daily-cleanup@host:prod-1` -| "Expired sessions purge" - -| Migration -| `migration:v1.0.0` -| "PostgreSQL migration from backup 2026-01-10" - -| Admin override -| `admin:jane@sudo` -| "Emergency data fix per ticket #INC-789" - -| AI agent -| `agent:document-classifier@model:v2` -| "Automated classification confidence=0.95" -|=== - -== Testing Migration - -=== Validation Queries - -[source,gql] ----- --- Compare row counts -SELECT COUNT(*) as lith_count FROM articles; --- Compare with: SELECT COUNT(*) FROM articles; (in PostgreSQL) - --- Verify all relationships migrated -SELECT - COUNT(*) as edge_count, - (SELECT COUNT(*) FROM articles WHERE author_id IS NOT NULL) as expected -FROM written_by; - --- Check for data integrity -SELECT * FROM articles -WHERE title IS NULL OR title = ''; - --- Verify provenance on migrated data -SELECT _provenance FROM articles LIMIT 5; ----- - -=== Regression Testing - -Create test queries that should return identical results: - -[source,javascript] ----- -// test/migration-validation.js -const testCases = [ - { - name: 'Article count by status', - postgres: 'SELECT status, COUNT(*) FROM articles GROUP BY status', - lith: 'SELECT status, COUNT(*) FROM articles GROUP BY status' - }, - { - name: 'Users with articles', - postgres: ` - SELECT u.name, COUNT(a.id) as article_count - FROM users u - LEFT JOIN articles a ON a.author_id = u.id - GROUP BY u.id - `, - lith: ` - TRAVERSE - START users - FOLLOW written_by DIRECTION INBOUND - RETURN users.name, COUNT(articles) as article_count - GROUP BY users._key - ` - } -]; - -async function validateMigration() { - for (const test of testCases) { - const pgResult = await postgres.query(test.postgres); - const lithResult = await lith.query(test.lith); - - assert.deepEqual( - normalizeResult(pgResult), - normalizeResult(lithResult), - `Mismatch in: ${test.name}` - ); - } -} ----- - -=== Data Sampling - -[source,gql] ----- --- Sample random documents to verify data integrity -SELECT * FROM articles -ORDER BY RANDOM() -LIMIT 10; - --- Check for missing references (should be 0) -SELECT COUNT(*) FROM written_by -WHERE _to NOT IN (SELECT _key FROM users); - --- Verify timestamps preserved -SELECT - MIN(created_at) as earliest, - MAX(created_at) as latest, - COUNT(*) as total -FROM articles; ----- - -== Rollback Planning - -Maintain parallel operation during migration: - -=== Dual-Write Strategy - -[source,javascript] ----- -// During migration: write to both systems -async function createArticle(data, context) { - // Write to Lith (new system) - const lithResult = await lith.insert('articles', data, context.provenance); - - // Write to PostgreSQL (old system) for rollback safety - await postgres.query( - 'INSERT INTO articles (id, title, content) VALUES ($1, $2, $3)', - [lithResult._key, data.title, data.content] - ); - - return lithResult; -} ----- - -=== Migration Phases - -[source] ----- -Phase 1: Preparation (1-2 weeks) -├── Schema translation complete -├── Export scripts tested -├── Import scripts tested -├── Application changes coded (feature flagged) -└── Rollback procedures documented - -Phase 2: Shadow Mode (1-2 weeks) -├── Dual-write enabled -├── Read from PostgreSQL (source of truth) -├── Write to both PostgreSQL and Lith -├── Compare query results continuously -└── Fix discrepancies - -Phase 3: Cutover (1 day) -├── Stop writes to PostgreSQL -├── Final sync to Lith -├── Verify data consistency -├── Switch reads to Lith -├── Enable Lith as source of truth -└── Keep PostgreSQL read-only for rollback - -Phase 4: Cleanup (1 week) -├── Monitor Lith in production -├── Remove dual-write code -├── Archive PostgreSQL backup -└── Decommission PostgreSQL (after confidence period) ----- - -== Migration Tooling - -=== Current Tools (Manual) - -[cols="1,2,2"] -|=== -| Tool | Purpose | Status - -| Export scripts -| Shell scripts for RDBMS export -| Examples in this guide - -| Schema translator -| SQL DDL → GQL DDL -| Manual process - -| Bulk import -| GQL `IMPORT FROM` command -| Available - -| Validation queries -| Data integrity checks -| Manual queries -|=== - -=== Planned Tools (Milestone M12) - -[cols="1,2,2"] -|=== -| Tool | Purpose | Timeline - -| `lith-migrate` -| CLI migration tool -| 🚧 Planned - -| PostgreSQL adapter -| Direct pg_dump integration -| 🚧 Planned - -| MySQL adapter -| Direct mysqldump integration -| 🚧 Planned - -| SQLite adapter -| Direct SQLite integration -| 🚧 Planned - -| Schema analyzer -| Automatic SQL→GQL translation -| 🚧 Planned - -| Relationship detector -| FK to edge auto-conversion -| 🚧 Planned - -| Validation framework -| Automated migration testing -| 🚧 Planned -|=== - -=== CLI Tool Design (Preview) - -[source,bash] ----- -# Planned lith-migrate CLI - -# Analyze source database -lith-migrate analyze \ - --source postgres://user:pass@host/db \ - --output analysis.json - -# Generate GQL schema -lith-migrate schema \ - --from analysis.json \ - --output schema.gql - -# Export data with transformation -lith-migrate export \ - --source postgres://user:pass@host/db \ - --format jsonl \ - --output ./export/ - -# Import to Lith -lith-migrate import \ - --target lith://localhost:8765/mydb \ - --schema schema.gql \ - --data ./export/ \ - --provenance '{"actor": "migration", "rationale": "Initial import"}' - -# Validate migration -lith-migrate validate \ - --source postgres://user:pass@host/db \ - --target lith://localhost:8765/mydb \ - --report validation-report.html ----- - -== Troubleshooting - -=== Common Migration Issues - -[cols="1,2,2"] -|=== -| Issue | Cause | Solution - -| **Type mismatch** -| PostgreSQL types not mapping cleanly -| Use explicit type conversion in export - -| **Missing relationships** -| Foreign keys not captured -| Re-run relationship export, check junction tables - -| **Duplicate keys** -| ID collision during import -| Use UUID generation, don't preserve integer IDs - -| **Provenance errors** -| Missing actor/rationale -| Set default provenance for bulk imports - -| **Slow imports** -| Large dataset, no batching -| Use batch imports with progress tracking - -| **Encoding issues** -| Non-UTF8 data in source -| Convert encoding during export - -| **NULL handling** -| Different NULL semantics -| Explicit NULL checks in validation -|=== - -=== Debugging Queries - -[source,gql] ----- --- Find documents without edges (orphaned data) -SELECT * FROM articles -WHERE _key NOT IN (SELECT _from FROM written_by); - --- Find edges pointing to non-existent documents -SELECT * FROM written_by -WHERE _to NOT IN (SELECT _key FROM users); - --- Check for duplicate data -SELECT title, COUNT(*) as count -FROM articles -GROUP BY title -HAVING COUNT(*) > 1; - --- Inspect journal for migration activity -SELECT * FROM _journal -WHERE provenance.actor LIKE 'migration%' -ORDER BY sequence DESC -LIMIT 100; ----- - -=== Performance Tuning - -[source,gql] ----- --- Create indexes for common query patterns -CREATE INDEX idx_articles_status ON articles(status); -CREATE INDEX idx_articles_created ON articles(created_at); - --- For edge traversals, ensure edge indexes -CREATE INDEX idx_written_by_from ON written_by(_from); -CREATE INDEX idx_written_by_to ON written_by(_to); - --- Analyze query performance -EXPLAIN SELECT * FROM articles WHERE status = 'published'; ----- - -== See Also - -* link:../QUICKSTART.adoc[Quickstart Guide] - Getting started with Lith -* link:../spec/gql.adoc[GQL Specification] - Complete GQL language reference -* link:API-REFERENCE.adoc[API Reference] - Programmatic interfaces -* link:DEPLOYMENT.adoc[Deployment Guide] - Production deployment -* link:../ARCHITECTURE.adoc[Architecture] - Technical architecture diff --git a/lithoglyph/docs/OBSERVABILITY.adoc b/lithoglyph/docs/OBSERVABILITY.adoc deleted file mode 100644 index eb57056c..00000000 --- a/lithoglyph/docs/OBSERVABILITY.adoc +++ /dev/null @@ -1,1440 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Observability Guide -:toc: left -:toclevels: 4 -:icons: font -:source-highlighter: rouge -:sectanchors: -:sectlinks: - -[.lead] -Comprehensive guide to monitoring, logging, tracing, metrics, alerting, backup, and operational visibility for Lith deployments. - -[NOTE] -==== -**Status: Documentation Complete, Implementation Planned** - -This document provides complete observability guidance. Full observability features are planned for Milestone M10. Currently, basic logging and journal introspection are available. -==== - -== Observability Philosophy - -Lith's audit-first design provides unique observability advantages: - -[cols="1,2,2"] -|=== -| Traditional DB | Lith | Advantage - -| Audit tables optional -| Journal built-in -| Complete operation history by default - -| "Who changed this?" = detective work -| Provenance on every mutation -| Actor + rationale always available - -| Schema history = migration files -| Self-normalizing with proofs -| Schema evolution is queryable - -| Metrics = external instrumentation -| Introspection is native -| Query the database about itself -|=== - -=== Three Pillars of Observability - -[cols="1,3"] -|=== -| Pillar | Lith Implementation - -| **Logs** -| Structured JSON logs with operation context, provenance, and timing - -| **Metrics** -| Prometheus-compatible metrics for queries, journal, storage, and constraints - -| **Traces** -| OpenTelemetry spans for distributed request tracing -|=== - -=== Built-In Observability - -Lith provides observability features that other databases require external tools for: - -[source,gql] ----- --- Query audit history (built-in) -SELECT * FROM _journal -WHERE collection = 'evidence' - AND provenance.actor = 'alice' -ORDER BY sequence DESC -LIMIT 100; - --- Schema evolution history (built-in) -SELECT * FROM _schema_history -WHERE collection = 'evidence' -ORDER BY version DESC; - --- Constraint violation history (built-in) -SELECT * FROM _constraint_violations -WHERE occurred_at > NOW() - INTERVAL '1 hour'; ----- - -== Logging - -=== Log Levels - -[cols="1,1,3"] -|=== -| Level | Numeric | Description - -| `FATAL` -| 0 -| Unrecoverable errors requiring immediate shutdown - -| `ERROR` -| 1 -| Recoverable errors, constraint violations, failed operations - -| `WARN` -| 2 -| Deprecation warnings, performance concerns, unusual conditions - -| `INFO` -| 3 -| Operations, lifecycle events, configuration changes - -| `DEBUG` -| 4 -| Detailed internal state, query plans, decision points - -| `TRACE` -| 5 -| Very verbose, individual block operations (development only) -|=== - -=== Log Format - -Lith uses structured JSON logging for machine-parseable output: - -.Standard Log Entry -[source,json] ----- -{ - "timestamp": "2026-01-12T10:30:45.123456Z", - "level": "INFO", - "component": "Form.Runtime", - "node_id": "lith-prod-01", - "request_id": "req_abc123def456", - "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", - "span_id": "00f067aa0ba902b7", - "message": "Query executed successfully", - "operation": "SELECT", - "collection": "evidence", - "documents_returned": 42, - "duration_ms": 12.5, - "actor": "user:alice@session:xyz", - "client_ip": "192.168.1.100" -} ----- - -.Error Log Entry -[source,json] ----- -{ - "timestamp": "2026-01-12T10:31:00.000000Z", - "level": "ERROR", - "component": "Form.Constraints", - "node_id": "lith-prod-01", - "request_id": "req_def789ghi012", - "message": "Constraint violation", - "operation": "INSERT", - "collection": "evidence", - "constraint": "score_range", - "constraint_type": "CHECK", - "violation_details": { - "field": "score", - "value": 150, - "expected": "0 <= score <= 100" - }, - "actor": "user:bob@session:abc", - "error_code": "CONSTRAINT_VIOLATION", - "error_id": 1003 -} ----- - -.Journal Operation Log -[source,json] ----- -{ - "timestamp": "2026-01-12T10:32:15.789012Z", - "level": "INFO", - "component": "Form.Journal", - "node_id": "lith-prod-01", - "message": "Journal entry written", - "sequence": 100542, - "operation": "UPDATE", - "collection": "evidence", - "document_key": "doc_12345", - "previous_sequence": 100541, - "block_id": 2048, - "provenance": { - "actor": "user:carol@session:mno", - "rationale": "Correcting source attribution per editor review" - }, - "has_inverse": true -} ----- - -=== Log Configuration - -.Configuration File (lith.toml) -[source,toml] ----- -[logging] -# Log level: fatal, error, warn, info, debug, trace -level = "info" - -# Output format: json, text, pretty -format = "json" - -# Output destination: stdout, stderr, file, syslog -output = "stdout" - -# File output settings (when output = "file") -[logging.file] -path = "/var/log/lith/lith.log" -max_size_mb = 100 -max_files = 10 -compress = true - -# Syslog settings (when output = "syslog") -[logging.syslog] -facility = "local0" -tag = "lith" - -# Component-specific log levels -[logging.components] -"Form.Runtime" = "info" -"Form.Journal" = "info" -"Form.Blocks" = "warn" -"Form.Constraints" = "debug" # More detail for constraint debugging -"Form.HTTP" = "info" -"Form.gRPC" = "info" - -# Sensitive data filtering -[logging.filter] -# Fields to redact from logs -redact_fields = ["password", "api_key", "token", "secret"] -# Truncate long values -max_value_length = 1000 ----- - -=== Log Destinations - -[cols="1,2,2"] -|=== -| Destination | Use Case | Configuration - -| **stdout/stderr** -| Container deployments (Docker, K8s) -| `output = "stdout"` - -| **File** -| Traditional server deployments -| `output = "file"` with rotation - -| **Syslog** -| Enterprise log aggregation -| `output = "syslog"` with facility - -| **Journald** -| systemd-based Linux systems -| `output = "journald"` -|=== - -=== Log Aggregation Integration - -==== Fluentd Configuration - -[source,yaml] ----- -# fluentd/lith.conf - - @type tail - path /var/log/lith/lith.log - pos_file /var/log/fluentd/lith.pos - tag lith - - @type json - time_key timestamp - time_format %Y-%m-%dT%H:%M:%S.%NZ - - - - - @type record_transformer - - environment production - service lith - - - - - @type elasticsearch - host elasticsearch.logging.svc.cluster.local - port 9200 - index_name lith-logs - type_name _doc - ----- - -==== Vector Configuration - -[source,toml] ----- -# vector/lith.toml -[sources.lith_logs] -type = "file" -include = ["/var/log/lith/*.log"] -read_from = "beginning" - -[transforms.lith_parse] -type = "remap" -inputs = ["lith_logs"] -source = ''' -. = parse_json!(.message) -.environment = "production" -''' - -[sinks.loki] -type = "loki" -inputs = ["lith_parse"] -endpoint = "http://loki:3100" -labels.service = "lith" -labels.component = "{{ component }}" -labels.level = "{{ level }}" - -[sinks.datadog] -type = "datadog_logs" -inputs = ["lith_parse"] -default_api_key = "${DATADOG_API_KEY}" ----- - -==== Loki Labels - -[source,yaml] ----- -# Recommended Loki labels for Lith logs -labels: - - service: lith - - environment: production - - node_id: lith-prod-01 - - component: Form.Runtime - - level: info - - operation: SELECT - - collection: evidence ----- - -== Metrics - -=== Prometheus Metrics - -Lith exports Prometheus-compatible metrics on the `/metrics` endpoint. - -==== Query Metrics - -[cols="1,1,3"] -|=== -| Metric | Type | Description - -| `lith_queries_total` -| Counter -| Total queries executed, labeled by operation and collection - -| `lith_query_duration_seconds` -| Histogram -| Query execution time distribution - -| `lith_query_rows_returned` -| Histogram -| Number of documents returned per query - -| `lith_query_errors_total` -| Counter -| Query errors by error type - -| `lith_query_in_flight` -| Gauge -| Currently executing queries -|=== - -[source,prometheus] ----- -# HELP lith_queries_total Total number of queries executed -# TYPE lith_queries_total counter -lith_queries_total{operation="SELECT",collection="evidence",status="success"} 15234 -lith_queries_total{operation="INSERT",collection="evidence",status="success"} 1234 -lith_queries_total{operation="INSERT",collection="evidence",status="error"} 12 - -# HELP lith_query_duration_seconds Query execution time -# TYPE lith_query_duration_seconds histogram -lith_query_duration_seconds_bucket{operation="SELECT",le="0.001"} 1000 -lith_query_duration_seconds_bucket{operation="SELECT",le="0.01"} 5000 -lith_query_duration_seconds_bucket{operation="SELECT",le="0.1"} 10000 -lith_query_duration_seconds_bucket{operation="SELECT",le="1"} 15000 -lith_query_duration_seconds_bucket{operation="SELECT",le="+Inf"} 15234 -lith_query_duration_seconds_sum{operation="SELECT"} 152.34 -lith_query_duration_seconds_count{operation="SELECT"} 15234 ----- - -==== Journal Metrics - -[cols="1,1,3"] -|=== -| Metric | Type | Description - -| `lith_journal_entries_total` -| Counter -| Total journal entries written - -| `lith_journal_sequence` -| Gauge -| Current journal sequence number - -| `lith_journal_write_duration_seconds` -| Histogram -| Journal write latency - -| `lith_journal_replay_entries` -| Counter -| Entries replayed during recovery - -| `lith_journal_size_bytes` -| Gauge -| Total journal size on disk -|=== - -==== Storage Metrics - -[cols="1,1,3"] -|=== -| Metric | Type | Description - -| `lith_storage_bytes_total` -| Gauge -| Total storage used - -| `lith_storage_blocks_total` -| Gauge -| Total blocks allocated - -| `lith_storage_blocks_free` -| Gauge -| Free blocks available - -| `lith_storage_collections` -| Gauge -| Number of collections - -| `lith_storage_documents_total` -| Gauge -| Total documents across all collections - -| `lith_storage_edges_total` -| Gauge -| Total edges across all edge collections -|=== - -==== Constraint Metrics - -[cols="1,1,3"] -|=== -| Metric | Type | Description - -| `lith_constraint_checks_total` -| Counter -| Total constraint evaluations - -| `lith_constraint_violations_total` -| Counter -| Constraint violation count by constraint name - -| `lith_constraint_check_duration_seconds` -| Histogram -| Constraint evaluation time -|=== - -==== Connection Metrics - -[cols="1,1,3"] -|=== -| Metric | Type | Description - -| `lith_connections_total` -| Counter -| Total connections opened - -| `lith_connections_active` -| Gauge -| Currently active connections - -| `lith_connections_idle` -| Gauge -| Idle connections in pool - -| `lith_connection_errors_total` -| Counter -| Connection errors by type -|=== - -=== Prometheus Configuration - -[source,yaml] ----- -# prometheus/prometheus.yml -scrape_configs: - - job_name: 'lith' - scrape_interval: 15s - static_configs: - - targets: - - 'lith-prod-01:9090' - - 'lith-prod-02:9090' - - 'lith-prod-03:9090' - relabel_configs: - - source_labels: [__address__] - target_label: instance - regex: '([^:]+):\d+' - replacement: '${1}' - metric_relabel_configs: - - source_labels: [__name__] - regex: 'lith_.*' - action: keep ----- - -=== Alert Rules - -[source,yaml] ----- -# prometheus/alerts/lith.yml -groups: - - name: lith - rules: - # High query latency - - alert: LithHighQueryLatency - expr: | - histogram_quantile(0.99, sum(rate(lith_query_duration_seconds_bucket[5m])) by (le)) > 1 - for: 5m - labels: - severity: warning - annotations: - summary: "High query latency detected" - description: "99th percentile query latency is {{ $value }}s" - - # High error rate - - alert: LithHighErrorRate - expr: | - sum(rate(lith_query_errors_total[5m])) / - sum(rate(lith_queries_total[5m])) > 0.05 - for: 5m - labels: - severity: critical - annotations: - summary: "High query error rate" - description: "Error rate is {{ $value | humanizePercentage }}" - - # Journal write failures - - alert: LithJournalWriteFailures - expr: | - increase(lith_journal_write_errors_total[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "Journal write failures detected" - description: "{{ $value }} journal write failures in last 5 minutes" - - # Storage space low - - alert: LithStorageSpaceLow - expr: | - (lith_storage_blocks_free / lith_storage_blocks_total) < 0.1 - for: 15m - labels: - severity: warning - annotations: - summary: "Low storage space" - description: "Only {{ $value | humanizePercentage }} storage remaining" - - # Connection pool exhaustion - - alert: LithConnectionPoolExhausted - expr: | - lith_connections_active / lith_connections_max > 0.9 - for: 5m - labels: - severity: warning - annotations: - summary: "Connection pool near exhaustion" - description: "{{ $value | humanizePercentage }} of connections in use" - - # Constraint violations spike - - alert: LithConstraintViolationSpike - expr: | - increase(lith_constraint_violations_total[5m]) > 100 - for: 5m - labels: - severity: warning - annotations: - summary: "High constraint violation rate" - description: "{{ $value }} constraint violations in 5 minutes" - - # Node down - - alert: LithNodeDown - expr: up{job="lith"} == 0 - for: 1m - labels: - severity: critical - annotations: - summary: "Lith node is down" - description: "Node {{ $labels.instance }} is not responding" ----- - -=== Grafana Dashboards - -==== Overview Dashboard - -[source,json] ----- -{ - "title": "Lith Overview", - "panels": [ - { - "title": "Query Rate", - "type": "graph", - "targets": [ - { - "expr": "sum(rate(lith_queries_total[1m])) by (operation)", - "legendFormat": "{{operation}}" - } - ] - }, - { - "title": "Query Latency (p99)", - "type": "graph", - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(lith_query_duration_seconds_bucket[5m])) by (le, operation))", - "legendFormat": "{{operation}}" - } - ] - }, - { - "title": "Error Rate", - "type": "singlestat", - "targets": [ - { - "expr": "sum(rate(lith_query_errors_total[5m])) / sum(rate(lith_queries_total[5m]))" - } - ], - "format": "percentunit" - }, - { - "title": "Active Connections", - "type": "gauge", - "targets": [ - { - "expr": "sum(lith_connections_active)" - } - ] - }, - { - "title": "Journal Sequence", - "type": "singlestat", - "targets": [ - { - "expr": "lith_journal_sequence" - } - ] - }, - { - "title": "Storage Used", - "type": "gauge", - "targets": [ - { - "expr": "sum(lith_storage_bytes_total)" - } - ], - "format": "bytes" - } - ] -} ----- - -==== Performance Dashboard Panels - -[source] ----- -Query Performance: -├── Query Rate by Operation (graph) -├── Query Latency Percentiles (graph: p50, p90, p99) -├── Slow Queries (table: queries > 100ms) -├── Query Errors by Type (graph) -└── Cache Hit Rate (gauge) - -Journal Performance: -├── Journal Write Rate (graph) -├── Journal Write Latency (graph) -├── Journal Size Growth (graph) -├── Sequence Number (counter) -└── Replay Progress (during recovery) - -Storage Performance: -├── Block Operations (graph: reads, writes) -├── Block I/O Latency (graph) -├── Storage Utilization (gauge) -├── Free Blocks Trend (graph) -└── Compaction Activity (graph) - -Constraint Performance: -├── Constraint Checks per Second (graph) -├── Constraint Check Latency (graph) -├── Violations by Constraint (bar chart) -└── Constraint Evaluation Cache Hit Rate (gauge) ----- - -== Tracing - -=== OpenTelemetry Integration - -Lith supports OpenTelemetry for distributed tracing. - -==== Configuration - -[source,toml] ----- -[tracing] -# Enable/disable tracing -enabled = true - -# Sampling rate (0.0 to 1.0) -sample_rate = 0.1 - -# Exporter: otlp, jaeger, zipkin -exporter = "otlp" - -# OTLP endpoint -[tracing.otlp] -endpoint = "http://otel-collector:4317" -protocol = "grpc" # or "http" - -# Jaeger endpoint (if exporter = "jaeger") -[tracing.jaeger] -endpoint = "http://jaeger:14268/api/traces" - -# Resource attributes -[tracing.resource] -service.name = "lith" -service.version = "0.0.2" -deployment.environment = "production" ----- - -==== Span Structure - -[source] ----- -Trace: lith.query -├── lith.query.parse (1ms) -│ └── Parse GQL query string -├── lith.query.plan (2ms) -│ └── Generate query execution plan -├── lith.query.validate (1ms) -│ └── Validate query against schema -├── lith.query.execute (50ms) -│ ├── lith.storage.read (45ms) -│ │ ├── lith.block.read (10ms) -│ │ ├── lith.block.read (10ms) -│ │ └── lith.block.read (10ms) -│ └── lith.filter (5ms) -└── lith.serialize (2ms) - └── Convert results to JSON ----- - -==== Trace Attributes - -[cols="1,2"] -|=== -| Attribute | Description - -| `db.system` -| Always "lith" - -| `db.operation` -| GQL operation (SELECT, INSERT, UPDATE, DELETE) - -| `db.collection` -| Target collection name - -| `db.statement` -| Sanitized GQL query (parameters removed) - -| `lith.sequence` -| Journal sequence number (for mutations) - -| `lith.actor` -| Provenance actor - -| `lith.documents_affected` -| Number of documents affected - -| `lith.constraint_checks` -| Number of constraints evaluated -|=== - -==== Context Propagation - -[source,rust] ----- -// Rust client example -use opentelemetry::trace::{Tracer, TraceContextExt}; -use lith::Client; - -async fn query_with_tracing(client: &Client, query: &str) -> Result> { - let tracer = global::tracer("my-application"); - - tracer.in_span("my_operation", |cx| async { - // Trace context is automatically propagated - let result = client.query(query).await?; - - // Add custom attributes - cx.span().set_attribute(KeyValue::new("app.query_purpose", "user_lookup")); - - Ok(result) - }).await -} ----- - -=== Jaeger UI Queries - -[source] ----- -# Find slow queries -service=lith operation=lith.query.execute minDuration=100ms - -# Find failed operations -service=lith error=true - -# Find queries for specific collection -service=lith db.collection=evidence - -# Find queries by actor -service=lith lith.actor=user:alice ----- - -== Health Checks - -=== Endpoints - -[cols="1,2,2"] -|=== -| Endpoint | Purpose | Response - -| `GET /health` -| Basic liveness -| `200 OK` if process is running - -| `GET /health/ready` -| Readiness to serve traffic -| `200 OK` if can accept queries - -| `GET /health/live` -| Kubernetes liveness probe -| `200 OK` if not deadlocked - -| `GET /health/startup` -| Kubernetes startup probe -| `200 OK` after initialization -|=== - -=== Detailed Health Response - -[source,json] ----- -{ - "status": "healthy", - "timestamp": "2026-01-12T10:30:00Z", - "version": "0.0.2", - "uptime_seconds": 86400, - "components": { - "journal": { - "status": "healthy", - "sequence": 100542, - "last_write": "2026-01-12T10:29:55Z", - "write_latency_ms": 2.5 - }, - "storage": { - "status": "healthy", - "blocks_total": 100000, - "blocks_free": 45000, - "utilization": 0.55 - }, - "connections": { - "status": "healthy", - "active": 25, - "idle": 75, - "max": 200 - }, - "memory": { - "status": "healthy", - "heap_used_bytes": 536870912, - "heap_total_bytes": 1073741824, - "gc_pause_ms": 1.2 - } - }, - "checks": { - "journal_writable": true, - "storage_writable": true, - "can_accept_queries": true - } -} ----- - -=== Kubernetes Probes - -[source,yaml] ----- -# kubernetes/lith-deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: lith -spec: - template: - spec: - containers: - - name: lith - image: lith:0.0.2 - ports: - - containerPort: 8765 - name: gql - - containerPort: 9090 - name: metrics - livenessProbe: - httpGet: - path: /health/live - port: 9090 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /health/ready - port: 9090 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 - startupProbe: - httpGet: - path: /health/startup - port: 9090 - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 30 ----- - -== Backup & Restore - -=== Backup Strategy - -Lith's append-only journal enables powerful backup capabilities: - -[cols="1,3"] -|=== -| Strategy | Description - -| **Full Backup** -| Complete database snapshot including journal, blocks, and metadata - -| **Incremental Backup** -| Only journal entries since last backup (extremely efficient) - -| **Continuous Archival** -| Stream journal to object storage in real-time - -| **Point-in-Time Recovery** -| Restore to any journal sequence number -|=== - -=== Backup Commands - -[source,bash] ----- -# Full backup to local directory -lith backup \ - --type full \ - --source /var/lib/lith/data \ - --destination /backup/lith-full-$(date +%Y%m%d) - -# Full backup to S3 -lith backup \ - --type full \ - --source /var/lib/lith/data \ - --destination s3://my-bucket/lith/full-$(date +%Y%m%d) - -# Incremental backup (since last full backup) -lith backup \ - --type incremental \ - --source /var/lib/lith/data \ - --since-sequence 100000 \ - --destination s3://my-bucket/lith/incr-$(date +%Y%m%d%H%M) - -# Continuous backup (streaming) -lith backup \ - --type continuous \ - --source /var/lib/lith/data \ - --destination s3://my-bucket/lith/journal/ \ - --daemon ----- - -=== Backup Configuration - -[source,toml] ----- -[backup] -# Enable automatic backups -enabled = true - -# Backup schedule -[backup.schedule] -# Full backup: weekly on Sunday at 2 AM -full = "0 2 * * 0" -# Incremental: hourly -incremental = "0 * * * *" - -# Backup destinations -[backup.destinations.primary] -type = "s3" -bucket = "lith-backups" -prefix = "production/" -region = "us-east-1" -storage_class = "STANDARD_IA" - -[backup.destinations.secondary] -type = "gcs" -bucket = "lith-backups-dr" -prefix = "production/" -region = "us-central1" - -# Retention policy -[backup.retention] -full_backups_keep = 4 # Keep 4 weekly full backups -incremental_keep_days = 7 # Keep 7 days of incrementals -journal_archive_days = 90 # Archive journal for 90 days - -# Encryption -[backup.encryption] -enabled = true -key_source = "kms" -kms_key_id = "alias/lith-backup-key" ----- - -=== Restore Procedures - -[source,bash] ----- -# Restore from full backup -lith restore \ - --source s3://my-bucket/lith/full-20260112 \ - --destination /var/lib/lith/data \ - --verify - -# Restore to specific point in time -lith restore \ - --source s3://my-bucket/lith/full-20260105 \ - --destination /var/lib/lith/data \ - --apply-incremental s3://my-bucket/lith/incr-* \ - --to-sequence 105000 - -# Restore to timestamp -lith restore \ - --source s3://my-bucket/lith/full-20260105 \ - --destination /var/lib/lith/data \ - --apply-incremental s3://my-bucket/lith/incr-* \ - --to-timestamp "2026-01-10T14:30:00Z" - -# Verify backup integrity -lith backup verify \ - --source s3://my-bucket/lith/full-20260112 \ - --checksum - -# List available restore points -lith restore list \ - --source s3://my-bucket/lith/ \ - --from 2026-01-01 \ - --to 2026-01-12 ----- - -=== Backup Verification - -[source,bash] ----- -# Verify backup integrity without restore -lith backup verify \ - --source s3://my-bucket/lith/full-20260112 \ - --deep # Verify all blocks and checksums - -# Test restore to temporary location -lith restore \ - --source s3://my-bucket/lith/full-20260112 \ - --destination /tmp/lith-test \ - --verify \ - --readonly - -# Compare backup with live database -lith backup compare \ - --backup s3://my-bucket/lith/full-20260112 \ - --live /var/lib/lith/data \ - --report /tmp/compare-report.json ----- - -== Disaster Recovery - -=== Recovery Objectives - -[cols="1,2,2"] -|=== -| Objective | Target | Implementation - -| **RPO** (Recovery Point Objective) -| < 1 hour -| Hourly incremental backups + continuous journal archival - -| **RTO** (Recovery Time Objective) -| < 4 hours -| Automated restore procedures + standby instances - -| **Data Integrity** -| Zero data loss for committed transactions -| Journal durability + backup verification -|=== - -=== DR Architecture - -[source] ----- -Primary Region (us-east-1) -├── Lith Primary (active) -│ ├── Journal → S3 (continuous backup) -│ └── Metrics → CloudWatch -├── Lith Replica (standby) -└── Backup Storage (S3) - ├── Full backups (weekly) - └── Incremental (hourly) - -DR Region (us-west-2) -├── Lith Standby (cold standby) -├── Backup Storage (S3, replicated) -└── DNS Failover (Route 53) ----- - -=== Failover Procedures - -.Automated Failover -[source,yaml] ----- -# Runbook: Automated Failover ---- -name: Lith Automated Failover -trigger: - - health_check_failed for 5m - - manual_trigger - -steps: - - name: Verify primary is truly down - command: lith health --target primary --timeout 30s - expect: failure - - - name: Stop writes to primary - command: lith drain --target primary - timeout: 60s - - - name: Promote replica to primary - command: lith promote --target replica - verify: lith health --target replica - - - name: Update DNS - command: | - aws route53 change-resource-record-sets \ - --hosted-zone-id $ZONE_ID \ - --change-batch file://failover-dns.json - - - name: Verify new primary - command: lith verify --target new-primary - expect: success - - - name: Notify on-call - command: pagerduty trigger --severity critical --message "Lith failover completed" ----- - -.Manual Recovery from Backup -[source,bash] ----- -#!/bin/bash -# manual-recovery.sh - Manual Lith recovery procedure - -set -euo pipefail - -BACKUP_SOURCE="s3://lith-backups/production/full-latest" -RESTORE_TARGET="/var/lib/lith/data" -TIMESTAMP="${1:-now}" - -echo "=== Lith Manual Recovery ===" -echo "Backup Source: $BACKUP_SOURCE" -echo "Target: $RESTORE_TARGET" -echo "Timestamp: $TIMESTAMP" - -# 1. Stop Lith if running -echo "Stopping Lith..." -systemctl stop lith || true - -# 2. Backup current state (if any) -if [ -d "$RESTORE_TARGET" ]; then - echo "Backing up current state..." - mv "$RESTORE_TARGET" "${RESTORE_TARGET}.pre-recovery-$(date +%s)" -fi - -# 3. Restore from backup -echo "Restoring from backup..." -lith restore \ - --source "$BACKUP_SOURCE" \ - --destination "$RESTORE_TARGET" \ - --to-timestamp "$TIMESTAMP" \ - --verify - -# 4. Verify restore -echo "Verifying restore..." -lith verify --data-dir "$RESTORE_TARGET" - -# 5. Start Lith -echo "Starting Lith..." -systemctl start lith - -# 6. Health check -echo "Running health check..." -sleep 10 -lith health --wait 60s - -echo "=== Recovery Complete ===" -lith status ----- - -=== DR Testing - -[source,toml] ----- -# DR test schedule (quarterly) -[dr_test] -schedule = "0 2 1 */3 *" # First day of each quarter at 2 AM - -[dr_test.scenarios] -# Test 1: Backup/Restore -backup_restore = """ -1. Create test data in isolated namespace -2. Perform full backup -3. Delete test data -4. Restore from backup -5. Verify data integrity -""" - -# Test 2: Failover -failover = """ -1. Simulate primary failure -2. Trigger automated failover -3. Verify replica becomes primary -4. Verify no data loss -5. Restore primary and fail back -""" - -# Test 3: Point-in-Time Recovery -pitr = """ -1. Create test data with known timestamp -2. Perform backup -3. Create more data -4. Restore to earlier timestamp -5. Verify only pre-timestamp data exists -""" ----- - -== Operational Runbooks - -=== Common Operations - -==== Investigating Slow Queries - -[source,gql] ----- --- 1. Check current slow queries -SELECT * FROM _slow_query_log -WHERE duration_ms > 100 - AND occurred_at > NOW() - INTERVAL '1 hour' -ORDER BY duration_ms DESC -LIMIT 20; - --- 2. Analyze query plan -EXPLAIN ANALYZE -SELECT * FROM evidence -WHERE source = 'leaked-document' - AND score > 80; - --- 3. Check index usage -INTROSPECT COLLECTION evidence; - --- 4. Review constraint overhead -SELECT * FROM _constraint_timing -WHERE collection = 'evidence' - AND check_duration_ms > 10; ----- - -==== Recovering from Constraint Violation Spike - -[source,bash] ----- -#!/bin/bash -# runbook: constraint-violation-spike.sh - -# 1. Identify which constraints are failing -lith query " - SELECT constraint_name, COUNT(*) as violations - FROM _constraint_violations - WHERE occurred_at > NOW() - INTERVAL '30 minutes' - GROUP BY constraint_name - ORDER BY violations DESC -" - -# 2. Get sample violations -lith query " - SELECT * - FROM _constraint_violations - WHERE constraint_name = '\$CONSTRAINT_NAME' - LIMIT 10 -" - -# 3. Identify actor patterns -lith query " - SELECT actor, COUNT(*) as violations - FROM _constraint_violations - WHERE occurred_at > NOW() - INTERVAL '30 minutes' - GROUP BY actor - ORDER BY violations DESC -" - -# 4. If caused by bad data, consider temporary constraint relaxation -# lith alter collection evidence DROP CONSTRAINT score_range; -# ... fix data ... -# lith alter collection evidence ADD CONSTRAINT score_range CHECK (score >= 0 AND score <= 100); ----- - -==== Journal Compaction - -[source,bash] ----- -#!/bin/bash -# runbook: journal-compaction.sh - -# 1. Check journal size -lith journal stats - -# 2. Estimate compaction impact -lith journal compact --dry-run --since 30d - -# 3. Perform compaction (during low-traffic window) -lith journal compact \ - --since 30d \ - --archive s3://lith-archive/journal/ \ - --verify - -# 4. Verify journal health after compaction -lith journal verify ----- - -==== Recovering from Corruption - -[source,bash] ----- -#!/bin/bash -# runbook: corruption-recovery.sh - -# 1. Stop Lith -systemctl stop lith - -# 2. Run integrity check -lith verify --data-dir /var/lib/lith/data --deep - -# 3. If corruption found, identify affected blocks -lith verify --data-dir /var/lib/lith/data --report /tmp/corruption-report.json - -# 4. Attempt repair from journal -lith repair \ - --data-dir /var/lib/lith/data \ - --replay-journal \ - --verify - -# 5. If repair fails, restore from backup -lith restore \ - --source s3://lith-backups/latest \ - --destination /var/lib/lith/data.restored \ - --verify - -# 6. Swap directories and restart -mv /var/lib/lith/data /var/lib/lith/data.corrupted -mv /var/lib/lith/data.restored /var/lib/lith/data -systemctl start lith ----- - -=== Capacity Planning - -[source,gql] ----- --- Storage growth analysis -SELECT - date_trunc('day', created_at) as day, - COUNT(*) as documents_added, - SUM(document_size_bytes) as bytes_added -FROM _storage_audit -WHERE created_at > NOW() - INTERVAL '30 days' -GROUP BY day -ORDER BY day; - --- Project storage needs -SELECT - current_size_gb, - daily_growth_gb, - current_size_gb + (daily_growth_gb * 90) as size_in_90_days, - current_size_gb + (daily_growth_gb * 365) as size_in_1_year -FROM ( - SELECT - SUM(size_bytes) / 1e9 as current_size_gb, - AVG(daily_growth_bytes) / 1e9 as daily_growth_gb - FROM _storage_summary -); - --- Query load projection -SELECT - date_trunc('day', timestamp) as day, - COUNT(*) as queries, - AVG(duration_ms) as avg_latency -FROM _query_log -WHERE timestamp > NOW() - INTERVAL '30 days' -GROUP BY day -ORDER BY day; ----- - -== See Also - -* link:DEPLOYMENT.adoc[Deployment Guide] - Production deployment -* link:SECURITY-AUTH.adoc[Security & Authentication] - Security hardening -* link:API-REFERENCE.adoc[API Reference] - Programmatic interfaces -* link:../ARCHITECTURE.adoc[Architecture] - Technical design -* link:../spec/journal.adoc[Journal Specification] - Journal format details diff --git a/lithoglyph/docs/SECURITY-AUTH.adoc b/lithoglyph/docs/SECURITY-AUTH.adoc deleted file mode 100644 index 80d5b383..00000000 --- a/lithoglyph/docs/SECURITY-AUTH.adoc +++ /dev/null @@ -1,1634 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Lith Security & Authentication Guide -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font -:sectnums: - -[.lead] -Security architecture, authentication models, authorization patterns, and hardening guidelines for Lith deployments. - -toc::[] - -== Overview - -Lith is designed with security as a foundational principle, not an afterthought. As an audit-grade database targeting investigative journalism, governance, and compliance workloads, security is integral to the narrative-first philosophy. - -[IMPORTANT] -==== -**Current Status: Pre-1.0 Development** - -Security features are being implemented incrementally. This document describes both current capabilities and planned features. Sections marked with 🚧 indicate planned functionality. -==== - -=== Security Philosophy - -Lith's security model follows these core principles: - -[cols="1,3"] -|=== -| Principle | Implementation - -| **Auditability First** -| Every operation is journaled with full provenance (actor, timestamp, rationale). Security events are immutable audit records. - -| **Defense in Depth** -| Multiple security layers: network, transport, authentication, authorization, data encryption, and application-level constraints. - -| **Fail Secure** -| When security checks fail, access is denied. No silent failures—all rejections include explanations. - -| **Least Privilege** -| Default to minimal permissions. Explicit grants required for each capability. - -| **Reversibility with Accountability** -| Operations can be undone, but the undo itself is recorded. Deletion is "soft" by default with provenance preserved. - -| **Constraint Enforcement** -| All constraints enforced at the Form.Bridge layer before data reaches storage. Cannot be bypassed by direct access. -|=== - -== Threat Model - -=== Assets to Protect - -[cols="1,2,2"] -|=== -| Asset | Sensitivity | Protection Requirements - -| **Document Data** -| Variable (public to highly sensitive) -| Access control, encryption at rest - -| **Edge Relationships** -| Often more sensitive than nodes -| Graph traversal controls - -| **Provenance Records** -| Critical for audit integrity -| Immutable, tamper-evident - -| **Journal Entries** -| Complete operation history -| Append-only, cryptographic chaining - -| **Schema Definitions** -| Business logic exposure -| Read-restricted by default - -| **Credentials** -| Authentication secrets -| Never stored, only hashed references - -| **Encryption Keys** -| Data confidentiality -| HSM integration, key rotation -|=== - -=== Threat Categories - -==== External Threats - -[cols="1,2,2"] -|=== -| Threat | Description | Mitigations - -| **Unauthorized Access** -| Attackers gaining system access -| Strong authentication, network controls, TLS - -| **Data Exfiltration** -| Bulk extraction of sensitive data -| Rate limiting, anomaly detection, query auditing - -| **Injection Attacks** -| GQL injection, parameter tampering -| Parameterized queries, input validation - -| **Man-in-the-Middle** -| Network traffic interception -| TLS 1.3 required, certificate pinning - -| **Denial of Service** -| Resource exhaustion attacks -| Rate limiting, query complexity limits -|=== - -==== Internal Threats - -[cols="1,2,2"] -|=== -| Threat | Description | Mitigations - -| **Insider Abuse** -| Authorized users exceeding permissions -| RBAC/ABAC, query auditing, anomaly detection - -| **Credential Compromise** -| Stolen or leaked credentials -| Short-lived tokens, MFA, credential rotation - -| **Configuration Errors** -| Misconfigured security settings -| Secure defaults, configuration validation - -| **Supply Chain** -| Compromised dependencies -| Dependency scanning, reproducible builds -|=== - -==== Data Integrity Threats - -[cols="1,2,2"] -|=== -| Threat | Description | Mitigations - -| **Tampering** -| Unauthorized data modification -| Cryptographic checksums, journal verification - -| **Repudiation** -| Denying performed actions -| Immutable provenance, signed operations - -| **Journal Manipulation** -| Altering audit history -| Append-only storage, Merkle tree verification -|=== - -== Authentication - -Authentication verifies the identity of actors (users, services, agents) before granting access. - -=== Actor Model - -Lith uses a unified actor model for all authentication: - -[source,gql] ----- --- Every operation requires an actor -INSERT INTO evidence { - title: "Document analysis", - content: "..." -} -WITH PROVENANCE { - actor: "user:alice@example.com", -- Who - rationale: "Initial case upload", -- Why - timestamp: NOW() -- When (auto-filled) -}; ----- - -Actor identifiers follow a URI scheme: - -[cols="1,2,2"] -|=== -| Scheme | Example | Use Case - -| `user:` -| `user:alice@example.com` -| Human users - -| `service:` -| `service:ingestion-pipeline` -| Automated services - -| `agent:` -| `agent:claude-research-assistant` -| AI agents - -| `system:` -| `system:scheduler` -| Internal system operations - -| `anonymous:` -| `anonymous:session-abc123` -| Unauthenticated access (if permitted) -|=== - -=== API Key Authentication - -Simple token-based authentication for programmatic access. - -==== Configuration - -[source,toml] ----- -[auth] -enabled = true -methods = ["api_key", "jwt"] - -[auth.api_key] -enabled = true -header_name = "X-Lith-API-Key" -# Alternative: query parameter (less secure, for debugging only) -query_param = "api_key" -query_param_enabled = false ----- - -==== Key Management - -[source,bash] ----- -# Generate a new API key -lith auth create-key \ - --name "ingestion-service" \ - --actor "service:ingestion-pipeline" \ - --permissions "collections:write,documents:create" \ - --expires "2025-12-31" - -# Output: -# API Key: lith_sk_live_abc123...xyz789 -# Key ID: key_01234567 -# WARNING: This key will only be shown once. Store it securely. - -# List keys (shows metadata only, not secrets) -lith auth list-keys - -# Revoke a key -lith auth revoke-key key_01234567 --reason "Rotation" ----- - -==== Usage - -[source,bash] ----- -# HTTP header (preferred) -curl -H "X-Lith-API-Key: lith_sk_live_abc123...xyz789" \ - https://lith.example.com/v1/query - -# In GQL session -lith shell --api-key "lith_sk_live_abc123...xyz789" ----- - -==== Key Scopes - -API keys support fine-grained permission scopes: - -[cols="1,2"] -|=== -| Scope | Permissions - -| `read` -| Read documents, edges, schemas - -| `write` -| Create, update documents and edges - -| `admin` -| Schema changes, collection management - -| `journal:read` -| Read journal entries - -| `introspect` -| INTROSPECT and EXPLAIN commands - -| `*` -| Full access (use sparingly) -|=== - -=== JWT Authentication - -Stateless token-based authentication with claims. - -==== Configuration - -[source,toml] ----- -[auth.jwt] -enabled = true -# Algorithm: RS256 (RSA), ES256 (ECDSA), or HS256 (HMAC, not recommended) -algorithm = "RS256" - -# Public key for verification (RS256/ES256) -public_key_file = "/etc/lith/jwt-public.pem" -# Or inline: -# public_key = """ -# -----BEGIN PUBLIC KEY----- -# ... -# -----END PUBLIC KEY----- -# """ - -# For HS256 (shared secret, not recommended for production) -# secret = "${JWT_SECRET}" - -# Token validation -issuer = "https://auth.example.com" -audience = "lith-api" -clock_skew_seconds = 30 - -# Claims mapping -claims.actor = "sub" # Subject claim → actor -claims.permissions = "scope" # Scope claim → permissions -claims.tenant = "tenant_id" # Custom claim → tenant ----- - -==== Token Structure - -Lith expects JWTs with these claims: - -[source,json] ----- -{ - "iss": "https://auth.example.com", - "sub": "user:alice@example.com", - "aud": "lith-api", - "exp": 1735689600, - "iat": 1735686000, - "scope": "read write", - "tenant_id": "acme-corp", - "lith": { - "collections": ["evidence", "sources"], - "max_query_depth": 5 - } -} ----- - -==== Usage - -[source,bash] ----- -# Obtain token from your identity provider -TOKEN=$(curl -s https://auth.example.com/oauth/token \ - -d "grant_type=client_credentials" \ - -d "client_id=$CLIENT_ID" \ - -d "client_secret=$CLIENT_SECRET" \ - | jq -r '.access_token') - -# Use with Lith -curl -H "Authorization: Bearer $TOKEN" \ - https://lith.example.com/v1/query ----- - -=== OIDC/OAuth2 Integration 🚧 - -Enterprise SSO integration via OpenID Connect. - -==== Configuration - -[source,toml] ----- -[auth.oidc] -enabled = true - -# Discovery endpoint (auto-configures from .well-known) -issuer = "https://login.example.com" -# Or manual configuration: -# authorization_endpoint = "https://login.example.com/oauth2/authorize" -# token_endpoint = "https://login.example.com/oauth2/token" -# userinfo_endpoint = "https://login.example.com/oauth2/userinfo" -# jwks_uri = "https://login.example.com/.well-known/jwks.json" - -client_id = "${OIDC_CLIENT_ID}" -client_secret = "${OIDC_CLIENT_SECRET}" - -# Scopes to request -scopes = ["openid", "profile", "email", "groups"] - -# Claims mapping -claims.actor = "email" -claims.groups = "groups" -claims.tenant = "organization" - -# Group-to-role mapping -[auth.oidc.role_mapping] -"lith-admins" = "admin" -"lith-editors" = "editor" -"lith-viewers" = "viewer" ----- - -==== Supported Providers - -[cols="1,2"] -|=== -| Provider | Notes - -| **Keycloak** -| Full support, recommended for self-hosted - -| **Auth0** -| Full support - -| **Okta** -| Full support - -| **Azure AD** -| Full support (configure tenant ID) - -| **Google Workspace** -| Supported via Google Identity Platform - -| **GitHub** -| OAuth2 only (not OIDC), limited claims -|=== - -=== Mutual TLS (mTLS) 🚧 - -Certificate-based authentication for service-to-service communication. - -==== Configuration - -[source,toml] ----- -[tls] -enabled = true -cert_file = "/etc/lith/server.crt" -key_file = "/etc/lith/server.key" - -[tls.mtls] -enabled = true -# Require client certificates -require_client_cert = true -# CA for validating client certificates -client_ca_file = "/etc/lith/client-ca.crt" - -# Certificate-to-actor mapping -[tls.mtls.mapping] -# Map certificate CN to actor -cn_pattern = "^(.+)@example\\.com$" -actor_template = "service:{1}" -# Or use certificate fingerprint -# fingerprint."SHA256:abc123..." = "service:trusted-ingester" ----- - -==== Certificate Requirements - -* X.509 v3 certificates -* RSA 2048-bit or ECDSA P-256 minimum -* Extended Key Usage: clientAuth -* Subject Alternative Name recommended - -=== FIDO2/WebAuthn 🚧 - -Passwordless authentication for web interfaces. - -[source,toml] ----- -[auth.webauthn] -enabled = true -rp_id = "lith.example.com" -rp_name = "Lith" -rp_origins = ["https://lith.example.com"] - -# Authenticator requirements -user_verification = "preferred" # required, preferred, discouraged -authenticator_attachment = "cross-platform" # platform, cross-platform -resident_key = "preferred" ----- - -=== Multi-Factor Authentication 🚧 - -Additional authentication factors for sensitive operations. - -[source,toml] ----- -[auth.mfa] -enabled = true -required_for = ["admin", "schema_changes", "bulk_delete"] - -[auth.mfa.totp] -enabled = true -issuer = "Lith" -digits = 6 -period = 30 - -[auth.mfa.webauthn] -enabled = true -# Reuse WebAuthn config ----- - -== Authorization - -Authorization determines what authenticated actors can do. - -=== Permission Model - -Lith uses a capability-based permission model: - -[source,text] ----- -: - -Examples: - documents:read - documents:create - documents:update - documents:delete - edges:create - edges:traverse - collections:create - collections:drop - schema:read - schema:modify - journal:read - journal:replay - admin:* ----- - -=== Role-Based Access Control (RBAC) - -Predefined roles simplify permission management. - -==== Built-in Roles - -[cols="1,2,3"] -|=== -| Role | Permissions | Use Case - -| `viewer` -| `documents:read`, `edges:read`, `schema:read` -| Read-only access - -| `editor` -| `viewer` + `documents:create,update`, `edges:create,update` -| Content creators - -| `analyst` -| `editor` + `edges:traverse`, `journal:read`, introspect -| Investigators, auditors - -| `admin` -| All permissions except `superuser:*` -| Database administrators - -| `superuser` -| All permissions including dangerous operations -| Emergency access only -|=== - -==== Custom Roles - -[source,gql] ----- --- Create a custom role -CREATE ROLE evidence_reviewer WITH { - permissions: [ - "documents:read", - "documents:update", - "edges:read", - "journal:read" - ], - description: "Can review and annotate evidence but not create" -} -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Role for external reviewers" -}; - --- Assign role to user -GRANT ROLE evidence_reviewer TO "user:reviewer@partner.org" -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Partner organization reviewer onboarding" -}; ----- - -==== Role Hierarchy - -[source,toml] ----- -[authorization.roles] -# Roles can inherit from others -[authorization.roles.senior_analyst] -inherits = ["analyst"] -additional_permissions = ["documents:delete", "edges:delete"] - -[authorization.roles.lead_investigator] -inherits = ["senior_analyst"] -additional_permissions = ["collections:create", "schema:modify"] ----- - -=== Attribute-Based Access Control (ABAC) 🚧 - -Fine-grained access control based on attributes. - -==== Policy Language - -[source,yaml] ----- -# /etc/lith/policies/evidence-access.yaml -policies: - - name: "evidence-classification-access" - description: "Restrict access based on classification level" - effect: "deny" - resources: - - "collection:evidence" - conditions: - - attribute: "document.classification" - operator: "in" - value: ["SECRET", "TOP_SECRET"] - - attribute: "actor.clearance" - operator: "less_than" - value: "document.classification" - - - name: "time-restricted-access" - description: "Restrict sensitive access to business hours" - effect: "deny" - resources: - - "collection:financial_records" - conditions: - - attribute: "request.time.hour" - operator: "not_between" - value: [9, 17] - - attribute: "request.time.day_of_week" - operator: "in" - value: ["Saturday", "Sunday"] - exceptions: - - role: "admin" - - role: "emergency_access" ----- - -==== Attribute Sources - -[cols="1,2,2"] -|=== -| Source | Attributes | Example - -| **Actor** -| From authentication claims -| `actor.department`, `actor.clearance` - -| **Resource** -| From document/edge metadata -| `document.classification`, `edge.sensitivity` - -| **Environment** -| Runtime context -| `request.time`, `request.ip`, `request.location` - -| **Relationship** -| Graph relationships -| `actor.owns(document)`, `actor.member_of(team)` -|=== - -=== Collection-Level Permissions - -[source,gql] ----- --- Set collection permissions -ALTER COLLECTION evidence SET PERMISSIONS { - read: ["role:analyst", "role:admin"], - write: ["role:editor", "role:admin"], - schema: ["role:admin"], - owner: "user:lead@example.com" -} -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Restrict evidence collection to authorized personnel" -}; ----- - -=== Row-Level Security 🚧 - -Filter documents based on actor attributes. - -[source,gql] ----- --- Create a row-level security policy -CREATE POLICY department_isolation ON evidence - FOR ALL - USING ( - document.department = actor.department - OR actor.role = 'admin' - ) -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "Users can only see documents in their department" -}; ----- - -=== Query Depth Limits - -Prevent excessive graph traversals: - -[source,toml] ----- -[authorization.query_limits] -# Maximum TRAVERSE depth -max_depth = 10 -# Maximum results per query -max_results = 10000 -# Query timeout -timeout_seconds = 30 - -# Per-role overrides -[authorization.query_limits.roles.analyst] -max_depth = 20 -max_results = 100000 -timeout_seconds = 300 ----- - -== Multi-Tenancy - -Lith supports multiple isolation models for multi-tenant deployments. - -=== Isolation Levels - -[cols="1,2,2,1"] -|=== -| Level | Description | Use Case | Overhead - -| **Database** -| Separate database files per tenant -| Highest security requirements -| High - -| **Collection** -| Shared database, separate collections -| Moderate isolation needs -| Medium - -| **Row** -| Shared collections with RLS policies -| Cost-optimized multi-tenancy -| Low -|=== - -=== Database-Level Isolation - -[source,toml] ----- -[tenancy] -mode = "database" -tenant_header = "X-Tenant-ID" -# Or from JWT claim: -# tenant_claim = "tenant_id" - -[tenancy.database] -# Directory structure: {data_dir}/{tenant_id}/ -data_dir = "/var/lib/lith/tenants" -# Create database on first access -auto_provision = true -# Template for new tenant databases -template_db = "/var/lib/lith/template.lith" ----- - -=== Collection-Level Isolation - -[source,toml] ----- -[tenancy] -mode = "collection" -tenant_header = "X-Tenant-ID" - -[tenancy.collection] -# Collection naming: {tenant_id}_{collection_name} -prefix_separator = "_" -# Prevent cross-tenant collection access -strict_isolation = true ----- - -=== Row-Level Isolation - -[source,toml] ----- -[tenancy] -mode = "row" -tenant_header = "X-Tenant-ID" - -[tenancy.row] -# Column containing tenant ID -tenant_column = "_tenant_id" -# Auto-inject tenant ID on writes -auto_inject = true -# Filter reads by tenant -auto_filter = true ----- - -== Data Security - -=== Encryption at Rest 🚧 - -Protect data stored on disk. - -[source,toml] ----- -[encryption] -enabled = true -algorithm = "AES-256-GCM" - -# Key management -[encryption.keys] -# Local key file (development only) -# key_file = "/etc/lith/master.key" - -# AWS KMS -provider = "aws-kms" -key_id = "arn:aws:kms:us-east-1:123456789:key/abc-123" - -# HashiCorp Vault -# provider = "vault" -# address = "https://vault.example.com" -# path = "secret/lith/master-key" -# token = "${VAULT_TOKEN}" - -# Key rotation -rotation_period_days = 90 -keep_old_keys = 3 ----- - -==== Block-Level Encryption - -[source,text] ----- -Block Header (64 bytes): -┌─────────────────────────────────────────────────────────────────┐ -│ magic (4) │ version (2) │ type (2) │ flags (4) │ ... │ -│ │ │ │ [encrypted bit set] │ -├─────────────────────────────────────────────────────────────────┤ -│ encryption_key_id (16 bytes) │ -├─────────────────────────────────────────────────────────────────┤ -│ nonce/IV (12 bytes) │ -└─────────────────────────────────────────────────────────────────┘ - -Block Payload (4032 bytes): -┌─────────────────────────────────────────────────────────────────┐ -│ AES-256-GCM encrypted content │ -│ + 16-byte authentication tag │ -└─────────────────────────────────────────────────────────────────┘ ----- - -=== Field-Level Encryption 🚧 - -Encrypt sensitive fields while keeping others queryable. - -[source,gql] ----- --- Define encrypted fields in schema -ALTER COLLECTION users ADD SCHEMA { - email: STRING, - ssn: ENCRYPTED STRING, -- Always encrypted - phone: ENCRYPTED STRING, -- Always encrypted - name: STRING -} -WITH PROVENANCE { - actor: "user:admin@example.com", - rationale: "PII protection compliance" -}; - --- Queries on encrypted fields require decryption key -SELECT name, DECRYPT(ssn) FROM users -WHERE id = "user_123" -WITH PROVENANCE { - actor: "user:compliance@example.com", - rationale: "Audit request #AR-2024-001" -}; ----- - -=== Encryption Key Management - -==== Key Hierarchy - -[source,text] ----- - ┌─────────────────┐ - │ Master Key │ (in HSM/KMS) - │ (KEK) │ - └────────┬────────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ ▼ - ┌────────────┐ ┌────────────┐ ┌────────────┐ - │ Block DEK │ │ Journal │ │ Field │ - │ │ │ DEK │ │ DEK │ - └────────────┘ └────────────┘ └────────────┘ - │ │ │ - ▼ ▼ ▼ - Block Data Journal Data Sensitive Fields ----- - -==== Key Rotation - -[source,bash] ----- -# Initiate key rotation -lith admin rotate-keys --type master - -# Status check -lith admin key-status - -# Output: -# Master Key: -# Current: key_v3 (created: 2024-06-01, expires: 2024-09-01) -# Previous: key_v2 (created: 2024-03-01, status: decrypt-only) -# Rotation: 45 days remaining -# -# Data Encryption Keys: -# Blocks: 1,234 re-encrypted, 56 pending -# Journal: Complete -# Fields: 789 re-encrypted, 12 pending ----- - -=== Data Masking 🚧 - -Mask sensitive data for non-privileged users. - -[source,gql] ----- --- Create masking policy -CREATE MASKING POLICY email_mask ON users.email - FOR SELECT - USING ( - CASE - WHEN actor.role = 'admin' THEN email - ELSE CONCAT(LEFT(email, 2), '***@***', RIGHT(email, 4)) - END - ) -WITH PROVENANCE { - actor: "user:security@example.com", - rationale: "GDPR email protection" -}; - --- Result for non-admin: --- al***@***.com ----- - -== Network Security - -=== TLS Configuration - -[source,toml] ----- -[tls] -enabled = true -min_version = "1.3" # TLS 1.3 required - -# Server certificate -cert_file = "/etc/lith/server.crt" -key_file = "/etc/lith/server.key" - -# Certificate chain (if using intermediate CA) -# ca_file = "/etc/lith/ca-chain.crt" - -# Cipher suites (TLS 1.3 uses fixed suites) -# For TLS 1.2 fallback (if enabled): -# cipher_suites = [ -# "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", -# "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" -# ] - -# OCSP stapling -ocsp_stapling = true - -# Session resumption (performance vs. forward secrecy trade-off) -session_tickets = false # Disable for maximum forward secrecy ----- - -=== Network Policies - -==== IP Allowlisting - -[source,toml] ----- -[network] -# Only accept connections from these CIDRs -allowed_cidrs = [ - "10.0.0.0/8", # Internal network - "192.168.1.0/24", # Office network - "203.0.113.50/32" # Specific host -] - -# Deny list takes precedence -denied_cidrs = [ - "10.0.99.0/24" # Untrusted subnet -] ----- - -==== Rate Limiting - -[source,toml] ----- -[network.rate_limiting] -enabled = true - -# Global limits -requests_per_second = 1000 -burst_size = 100 - -# Per-IP limits -per_ip_requests_per_second = 100 -per_ip_burst_size = 20 - -# Per-actor limits (authenticated) -per_actor_requests_per_second = 500 -per_actor_burst_size = 50 - -# Exempt certain IPs (monitoring, health checks) -exempt_cidrs = ["10.0.0.1/32"] ----- - -==== Connection Limits - -[source,toml] ----- -[network.connections] -# Maximum concurrent connections -max_connections = 10000 - -# Per-IP connection limit -max_connections_per_ip = 100 - -# Idle timeout -idle_timeout_seconds = 300 - -# Read/write timeouts -read_timeout_seconds = 30 -write_timeout_seconds = 30 ----- - -=== Kubernetes Network Policies - -[source,yaml] ----- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: lith-network-policy - namespace: lith -spec: - podSelector: - matchLabels: - app: lith - policyTypes: - - Ingress - - Egress - ingress: - # Allow from application pods - - from: - - namespaceSelector: - matchLabels: - name: application - podSelector: - matchLabels: - needs-lith: "true" - ports: - - protocol: TCP - port: 5432 - # Allow from monitoring - - from: - - namespaceSelector: - matchLabels: - name: monitoring - ports: - - protocol: TCP - port: 9090 # Metrics - egress: - # Allow DNS - - to: - - namespaceSelector: {} - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - protocol: UDP - port: 53 - # Allow to backup storage - - to: - - ipBlock: - cidr: 10.0.100.0/24 - ports: - - protocol: TCP - port: 443 ----- - -== Audit Logging - -=== Audit Events - -All security-relevant events are logged to the journal and optionally to external systems. - -==== Event Categories - -[cols="1,2,2"] -|=== -| Category | Events | Example - -| **Authentication** -| Login, logout, token refresh, MFA -| `auth.login.success`, `auth.mfa.required` - -| **Authorization** -| Permission checks, role changes -| `authz.denied`, `authz.role.granted` - -| **Data Access** -| Queries, reads, traversals -| `data.query`, `data.traverse` - -| **Data Modification** -| Creates, updates, deletes -| `data.create`, `data.delete` - -| **Schema Changes** -| Collection/schema modifications -| `schema.alter`, `collection.create` - -| **Administration** -| Config changes, user management -| `admin.config.change`, `admin.user.create` - -| **Security** -| Encryption, key operations -| `security.key.rotate`, `security.encrypt` -|=== - -=== Audit Log Format - -[source,json] ----- -{ - "timestamp": "2024-06-15T14:30:22.123Z", - "event_id": "evt_abc123", - "category": "data.query", - "severity": "info", - "actor": { - "type": "user", - "id": "user:alice@example.com", - "ip": "192.168.1.100", - "user_agent": "lith-cli/0.1.0" - }, - "resource": { - "type": "collection", - "id": "evidence", - "path": "/collections/evidence" - }, - "action": { - "type": "SELECT", - "query_hash": "sha256:abc123...", - "result_count": 42, - "duration_ms": 150 - }, - "context": { - "session_id": "sess_xyz789", - "request_id": "req_def456", - "tenant_id": "acme-corp" - }, - "provenance": { - "rationale": "Case review for investigation #INV-2024-100", - "journal_seq": 12345 - } -} ----- - -=== External Audit Destinations 🚧 - -[source,toml] ----- -[audit] -enabled = true -# Always write to journal -journal = true - -# External destinations -[audit.syslog] -enabled = true -address = "syslog.example.com:514" -protocol = "tcp" -format = "rfc5424" - -[audit.splunk] -enabled = true -hec_url = "https://splunk.example.com:8088/services/collector" -token = "${SPLUNK_HEC_TOKEN}" -index = "lith_audit" - -[audit.elasticsearch] -enabled = true -nodes = ["https://es1.example.com:9200", "https://es2.example.com:9200"] -index = "lith-audit" -username = "${ES_USERNAME}" -password = "${ES_PASSWORD}" - -[audit.s3] -enabled = true -bucket = "lith-audit-logs" -prefix = "audit/" -region = "us-east-1" -# Rotate files hourly -rotation_period = "1h" ----- - -== Secrets Management - -=== Environment Variables - -[source,bash] ----- -# Secrets should be injected via environment -export LITH_AUTH_JWT_SECRET="..." -export LITH_ENCRYPTION_MASTER_KEY="..." -export LITH_OIDC_CLIENT_SECRET="..." ----- - -=== HashiCorp Vault Integration 🚧 - -[source,toml] ----- -[secrets] -provider = "vault" - -[secrets.vault] -address = "https://vault.example.com" -auth_method = "kubernetes" # or "token", "approle" - -# Kubernetes auth -kubernetes_role = "lith" -kubernetes_mount = "kubernetes" - -# Secret paths -[secrets.vault.paths] -master_key = "secret/data/lith/master-key" -jwt_secret = "secret/data/lith/jwt-secret" -oidc_secret = "secret/data/lith/oidc-client" - -# Dynamic secrets for database connections -# postgres_creds = "database/creds/lith-role" ----- - -=== AWS Secrets Manager Integration 🚧 - -[source,toml] ----- -[secrets] -provider = "aws-secrets-manager" - -[secrets.aws] -region = "us-east-1" - -[secrets.aws.paths] -master_key = "lith/master-key" -jwt_secret = "lith/jwt-secret" -oidc_secret = "lith/oidc-client" ----- - -== Security Hardening - -=== Operating System - -[source,bash] ----- -# Run as non-root user -useradd -r -s /sbin/nologin lith -chown -R lith:lith /var/lib/lith - -# Restrict file permissions -chmod 700 /var/lib/lith -chmod 600 /etc/lith/lith.toml -chmod 400 /etc/lith/*.key - -# Limit capabilities -setcap 'cap_net_bind_service=+ep' /usr/bin/lith # If binding to port <1024 ----- - -=== Systemd Hardening - -[source,ini] ----- -[Service] -User=lith -Group=lith - -# Security hardening -NoNewPrivileges=yes -PrivateTmp=yes -PrivateDevices=yes -ProtectSystem=strict -ProtectHome=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -ReadWritePaths=/var/lib/lith -ReadOnlyPaths=/etc/lith - -# Restrict capabilities -CapabilityBoundingSet= -AmbientCapabilities= - -# Restrict system calls -SystemCallFilter=@system-service -SystemCallErrorNumber=EPERM - -# Memory protection -MemoryDenyWriteExecute=yes - -# Network -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX ----- - -=== Container Hardening - -[source,dockerfile] ----- -FROM gcr.io/distroless/cc-debian12:nonroot - -# Run as non-root -USER 65532:65532 - -# Read-only root filesystem -# (configured in orchestrator, not Dockerfile) - -COPY --chown=65532:65532 lith /usr/bin/lith - -ENTRYPOINT ["/usr/bin/lith"] ----- - -[source,yaml] ----- -# Kubernetes security context -securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault ----- - -=== Query Injection Prevention - -Lith uses parameterized queries to prevent GQL injection: - -[source,gql] ----- --- WRONG: String interpolation (vulnerable) --- SELECT * FROM users WHERE name = '{user_input}' - --- CORRECT: Parameterized query -SELECT * FROM users WHERE name = $name -WITH PARAMS { - name: @user_input -}; ----- - -The Form.Bridge layer validates all inputs before execution. - -== Compliance - -=== GDPR - -Lith's provenance system supports GDPR requirements: - -[cols="1,2"] -|=== -| Requirement | Lith Feature - -| **Right to Access** -| `INTROSPECT HISTORY document_id` returns all changes with provenance - -| **Right to Rectification** -| Updates preserve history; provenance records correction reason - -| **Right to Erasure** -| Soft delete with provenance; hard delete available with audit trail - -| **Data Portability** -| Export entire collection with provenance as JSON - -| **Purpose Limitation** -| Provenance `rationale` field documents processing purpose -|=== - -[source,gql] ----- --- GDPR data export -SELECT * FROM users WHERE id = "user_123" -WITH PROVENANCE_HISTORY -EXPORT FORMAT JSON -WITH PROVENANCE { - actor: "user:dpo@example.com", - rationale: "GDPR Article 15 access request #SAR-2024-001" -}; - --- GDPR erasure with audit trail -DELETE FROM users WHERE id = "user_123" -WITH PROVENANCE { - actor: "user:dpo@example.com", - rationale: "GDPR Article 17 erasure request #ER-2024-042" -}; --- Document is soft-deleted; provenance preserved for audit ----- - -=== SOC 2 🚧 - -[cols="1,2"] -|=== -| Control | Lith Support - -| **Access Control (CC6.1)** -| RBAC/ABAC, authentication, authorization - -| **Logical Access (CC6.2)** -| Multi-tenancy, collection permissions, RLS - -| **System Operations (CC7.1)** -| Health checks, monitoring, alerting - -| **Change Management (CC8.1)** -| Schema versioning, migration tracking - -| **Risk Mitigation (CC9.1)** -| Encryption, backup, disaster recovery - -| **Logging (CC7.2)** -| Immutable journal, audit events -|=== - -=== HIPAA 🚧 - -[cols="1,2"] -|=== -| Safeguard | Lith Support - -| **Access Controls** -| Authentication, authorization, session management - -| **Audit Controls** -| Immutable journal with provenance - -| **Integrity Controls** -| CRC32C checksums, Merkle tree verification - -| **Transmission Security** -| TLS 1.3, mTLS - -| **Encryption** -| At-rest and field-level encryption -|=== - -== Security Operations - -=== Vulnerability Scanning - -[source,bash] ----- -# Scan Lith binary -trivy fs /usr/bin/lith - -# Scan container image -trivy image lith:latest - -# Scan dependencies (Forth/Zig) -# (Manual review required - limited automated tooling) - -# Check for CVEs affecting Lith -lith admin security-check ----- - -=== Incident Response - -==== Detection - -[source,gql] ----- --- Query for suspicious activity -SELECT * FROM $journal -WHERE timestamp > NOW() - INTERVAL '1 hour' - AND ( - event_type = 'auth.login.failed' - OR event_type = 'authz.denied' - OR event_type LIKE 'admin.%' - ) -ORDER BY timestamp DESC -WITH PROVENANCE { - actor: "service:security-monitor", - rationale: "Automated security monitoring" -}; ----- - -==== Response Procedures - -[source,bash] ----- -# Emergency: Disable all external access -lith admin emergency-lockdown --reason "Security incident" - -# Revoke all active sessions -lith auth revoke-all-sessions --except-admin - -# Export forensic evidence -lith journal export \ - --since "2024-06-15T00:00:00Z" \ - --output incident-journal.jsonl - -# Rotate compromised keys -lith admin rotate-keys --all --reason "Incident response" ----- - -=== Security Monitoring - -[source,yaml] ----- -# Prometheus alerts for security events -groups: - - name: lith-security - rules: - - alert: HighAuthFailureRate - expr: rate(lith_auth_failures_total[5m]) > 10 - for: 2m - labels: - severity: warning - annotations: - summary: "High authentication failure rate" - - - alert: UnauthorizedAccessAttempts - expr: rate(lith_authz_denied_total[5m]) > 5 - for: 1m - labels: - severity: critical - annotations: - summary: "Multiple unauthorized access attempts" - - - alert: SuspiciousQueryPattern - expr: lith_query_complexity > 1000 - for: 1m - labels: - severity: warning - annotations: - summary: "Unusually complex query detected" ----- - -== Security Checklist - -=== Pre-Production Checklist - -[cols="1,1,3"] -|=== -| Category | Check | Action - -| **TLS** -| [ ] -| TLS 1.3 enabled, valid certificates, no self-signed in production - -| **Authentication** -| [ ] -| At least one auth method configured, API keys rotated - -| **Authorization** -| [ ] -| RBAC configured, no default admin passwords - -| **Encryption** -| [ ] -| Encryption at rest enabled, keys in HSM/KMS - -| **Network** -| [ ] -| Firewall configured, unnecessary ports closed - -| **Logging** -| [ ] -| Audit logging enabled, logs shipped externally - -| **Backups** -| [ ] -| Automated backups, tested restore procedure - -| **Secrets** -| [ ] -| No secrets in config files, using secrets manager - -| **Updates** -| [ ] -| Patch management process defined -|=== - -=== Periodic Review Checklist - -[cols="1,1,3"] -|=== -| Frequency | Check | Action - -| **Weekly** -| [ ] -| Review authentication failures, check for anomalies - -| **Monthly** -| [ ] -| Rotate API keys, review access permissions - -| **Quarterly** -| [ ] -| Rotate encryption keys, conduct access review - -| **Annually** -| [ ] -| Full security audit, penetration testing -|=== - -== Security Reporting - -Found a security vulnerability? See link:../SECURITY.md[SECURITY.md] for responsible disclosure procedures. - -**Do not report security vulnerabilities through public GitHub issues.** - -== See Also - -* link:../SECURITY.md[Security Policy] - Vulnerability reporting -* link:DEPLOYMENT.adoc[Deployment Guide] - Production hardening -* link:OBSERVABILITY.adoc[Observability Guide] - Security monitoring -* link:../ARCHITECTURE.adoc[Architecture] - Security architecture overview diff --git a/lithoglyph/docs/WP05_interactive_documentary.md b/lithoglyph/docs/WP05_interactive_documentary.md deleted file mode 100644 index d16da795..00000000 --- a/lithoglyph/docs/WP05_interactive_documentary.md +++ /dev/null @@ -1,1062 +0,0 @@ -# White Paper 05: Lith for Interactive Documentary and Journalism - -**Status**: Draft -**Version**: 0.1.0 -**Date**: 2025-01-11 -**Authors**: Jonathan D.A. Jewell, Claude (Anthropic) -**License**: MPL-2.0 - -## Abstract - -Interactive documentary (i-doc) represents a paradigm shift from linear narrative to reader-driven navigation through evidence. However, i-docs lack appropriate database infrastructure—traditional databases cannot track the provenance, corrections, and multi-perspective navigation that journalism requires. This white paper presents Lith as purpose-built infrastructure for i-docs, demonstrating how narrative-first database design enables epistemological transparency, audience-specific navigation (boundary objects), and correction workflows essential for journalism in the post-truth era. - -## 1. Introduction - -### 1.1 The i-doc Movement - -Interactive documentary emerged in the 2010s as a response to passive media consumption. As defined by MIT's Open Doc Lab (Aston & Gaudenzi, 2012): - -> **i-doc**: "Any project that starts with an intention to document the 'real' and that uses digital interactive technology to realize this intention." - -Key principles: -- **Navigation over narration**: Readers choose their own path through evidence -- **Multiple entry points**: Different audiences access the same evidence differently -- **Transparency**: Epistemological foundations are visible, not hidden -- **Living documents**: Content updates, corrections, retractions over time - -**Examples**: -- *Hollow* (Elaine McMillion Sheldon): Multi-perspective rural documentary -- *Fort McMoney* (David Dufresne): Interactive journalism game -- *Gaza/Sderot* (Bruno + Arte): Dual-perspective conflict coverage - -### 1.2 The Database Problem - -i-docs require databases that can: - -1. **Track evidence provenance**: "Where did this claim come from?" -2. **Support corrections transparently**: Retractions, updates, clarifications -3. **Enable multi-perspective navigation**: Same evidence, different audience paths -4. **Maintain epistemological metadata**: Quality scores, verification status -5. **Preserve narrative context**: Why was this evidence included? By whom? When? - -**Traditional databases fail** because they: -- Treat history as write-only audit logs (not queryable narrative) -- Lack semantic distinction between data changes and corrections -- Cannot represent "why" a connection exists between evidence and claims -- Require application-level code for provenance tracking - -### 1.3 Lith as i-doc Infrastructure - -Lith was designed for exactly this use case: - -| i-doc Requirement | Lith Feature | -|-------------------|----------------| -| Evidence provenance | `[PROVENANCE]` queries, journal entries | -| Transparent corrections | Reversible operations with rationale | -| Multi-perspective navigation | Boundary object collections (navigation paths) | -| Epistemological metadata | PROMPT scores as first-class data | -| Narrative context | Constraints-as-ethics, explainable operations | - -**Thesis**: Lith is the natural database for i-docs because it treats the database itself as part of the documentary story. - -## 2. i-doc Theory: Navigation Over Narration - -### 2.1 Reader Agency - -Traditional journalism: -``` -Journalist → Article → Reader (passive) -``` - -i-doc: -``` -Journalist → Evidence Graph → Reader (active navigator) -``` - -**Example** (UK Inflation 2023): - -Traditional article: -> "Inflation in 2023 disproportionately affected renters, with rent costs rising 12% compared to 8% overall inflation, according to ONS data." - -i-doc approach: -- **Claim**: "Inflation disproportionately affected renters" -- **Evidence**: ONS CPI data, academic study, think tank report -- **Relationships**: Which evidence supports/contradicts/contextualizes? -- **Navigation paths**: - - Skeptic → Start with methodology - - Policymaker → Start with authoritative sources - - Affected person → Start with personal impact stories - -Readers choose their path based on their needs. - -### 2.2 Boundary Objects (Star & Griesemer, 1989) - -**Definition**: Objects that inhabit multiple social worlds and satisfy the informational requirements of each. - -In i-docs, **the same evidence serves multiple audiences**: - -| Evidence | Researcher Perspective | Policymaker Perspective | Affected Person Perspective | -|----------|------------------------|-------------------------|------------------------------| -| ONS CPI Data | Methodology → Replicability → Raw data | Authority → Summary stats → Recommendations | "What does this mean for me?" → Rent specifically | -| Academic Study | Peer review status → Citations → Methods | Policy implications → Summary | Readability → Plain language | -| Expert Interview | Credentials → Bias disclosure → Full transcript | Authority → Key quotes | Relatability → Human story | - -Lith enables this via **navigation path collections** (Section 4). - -### 2.3 Epistemological Transparency - -i-docs must make their epistemology **visible and queryable**: - -**Bad** (opaque): -> "Sources confirm inflation affected renters more." - -**Good** (transparent): -> "This claim is supported by: -> - ONS CPI data (PROMPT score: 97.5/100, provenance: 100, replicability: 100) -> - Academic study (PROMPT: 81.8, peer-reviewed, n=5000 households) -> - Think tank report (PROMPT: 72.3, methodology transparent but non-peer-reviewed) -> -> Counterpoint from landlords' association (PROMPT: 59, expert interview, not replicated)" - -Lith makes this queryable: -```gql --- Get all evidence supporting a claim, sorted by PROMPT overall score -SELECT evidence.title, evidence.prompt_overall, - relationship.weight, relationship.reasoning -FROM claims - JOIN relationships ON claims.id = relationships.from_id - JOIN evidence ON relationships.to_id = evidence.id -WHERE claims.id = 'claim_renters_disproportionate' -ORDER BY evidence.prompt_overall DESC; -``` - -## 3. The PROMPT Framework in Lith - -### 3.1 PROMPT Dimensions - -PROMPT (Provenance, Replicability, Objective, Methodology, Publication, Transparency) is a 6-dimensional framework for scoring evidence quality (Wineburg et al., 2022). - -Lith stores PROMPT scores as **narrative metadata**: - -```gql -CREATE COLLECTION evidence ( - id UUID PRIMARY KEY, - title VARCHAR NOT NULL, - evidence_type VARCHAR NOT NULL, - prompt_scores STRUCT { - provenance INT, -- 0-100: Source authority/chain of custody - replicability INT, -- 0-100: Reproducibility of findings - objective INT, -- 0-100: Bias/conflicts of interest - methodology INT, -- 0-100: Research quality/rigor - publication INT, -- 0-100: Peer review/editorial standards - transparency INT, -- 0-100: Data/methods availability - overall COMPUTED AS AVG(all above) - }, - added_by VARCHAR, - added_at TIMESTAMP, - url TEXT, - zotero_key VARCHAR -) WITH RATIONALE; -``` - -**Example** (from BoFIG UK Inflation dataset): - -| Evidence | Prov | Repl | Obj | Meth | Pub | Trans | Overall | -|----------|------|------|-----|------|-----|-------|---------| -| ONS CPI Data | 100 | 100 | 95 | 95 | 100 | 95 | 97.5 | -| Academic Study (peer-reviewed) | 85 | 80 | 75 | 85 | 90 | 75 | 81.8 | -| Think Tank Report | 75 | 70 | 65 | 75 | 80 | 70 | 72.3 | -| Expert Interview | 85 | 45 | 60 | 50 | 40 | 75 | 59.0 | - -### 3.2 PROMPT Score Evolution - -Scores change over time (retractions, replication failures): - -```gql --- Initial scoring -INSERT INTO evidence (title, prompt_scores) -VALUES ('Climate Study X', { - provenance: 90, - replicability: 85, - objective: 80, - methodology: 90, - publication: 95, - transparency: 85 -}) -SCORED_BY "journalist_jane" -RATIONALE "Peer-reviewed in Nature, strong methodology, data available"; - --- Later: Replication failure -UPDATE evidence -SET prompt_scores.replicability = 30, - prompt_scores.overall = RECOMPUTE -WHERE id = 'climate_study_x' -REASON "Study failed to replicate per Science retraction notice 2024-03-15" -RETRACTION_URL "https://doi.org/10.1126/science.retraction.2024.03" -DISCLOSED_BY "editor_bob" -DISCLOSED_AT "2024-03-16T09:00:00Z"; - --- Lith journals this as a correction, preserving original scores in provenance -``` - -**Agents/readers can query score history**: -```gql -INTROSPECT evidence.climate_study_x PROMPT_HISTORY; - --- Returns: --- [ --- { date: "2024-01-15", overall: 87.5, scored_by: "jane", --- note: "Initial scoring" }, --- { date: "2024-03-16", overall: 67.5, scored_by: "bob", --- note: "Replication failure", replicability: 85→30 } --- ] -``` - -### 3.3 Audience-Weighted PROMPT - -Different audiences prioritize different dimensions: - -| Audience | Top Priority | Secondary | Tertiary | -|----------|--------------|-----------|----------| -| Researcher | Methodology, Replicability | Transparency | Publication | -| Policymaker | Provenance, Publication | Objective | Methodology | -| Skeptic | Objective, Transparency | Replicability | Provenance | -| Affected Person | Transparency, Objective | Provenance | (simplicity) | - -Lith supports audience-specific scoring: -```gql -SELECT evidence.title, - audience_weighted_prompt(evidence.prompt_scores, 'RESEARCHER') AS researcher_score, - audience_weighted_prompt(evidence.prompt_scores, 'SKEPTIC') AS skeptic_score -FROM evidence -WHERE investigation = 'uk_inflation_2023'; - --- Custom weighting function: -CREATE FUNCTION audience_weighted_prompt(scores, audience) AS - CASE audience - WHEN 'RESEARCHER' THEN - 0.30 * scores.methodology + 0.30 * scores.replicability + - 0.20 * scores.transparency + 0.20 * scores.publication - WHEN 'SKEPTIC' THEN - 0.35 * scores.objective + 0.30 * scores.transparency + - 0.20 * scores.replicability + 0.15 * scores.provenance - -- ... etc - END; -``` - -## 4. Boundary Objects: Navigation Paths - -### 4.1 Implementation in Lith - -```gql -CREATE COLLECTION navigation_paths ( - id UUID PRIMARY KEY, - name VARCHAR NOT NULL, - investigation_id VARCHAR NOT NULL, - audience_type ENUM('RESEARCHER', 'POLICYMAKER', 'SKEPTIC', 'AFFECTED_PERSON') NOT NULL, - description TEXT, - created_by VARCHAR, - created_at TIMESTAMP -) WITH RATIONALE; - -CREATE COLLECTION path_nodes ( - path_id UUID REFERENCES navigation_paths(id), - entity_id UUID NOT NULL, -- claim_id or evidence_id - entity_type ENUM('CLAIM', 'EVIDENCE') NOT NULL, - order INT NOT NULL, - context TEXT, -- Why this node is in this position for this audience - PRIMARY KEY (path_id, order) -); -``` - -### 4.2 Example: UK Inflation 2023 - -**Researcher Path**: -```gql -CREATE NAVIGATION_PATH 'researcher_path_inflation' -FOR INVESTIGATION 'uk_inflation_2023' -AUDIENCE 'RESEARCHER' -BEGIN - -- Start with methodology - NODE evidence WHERE evidence_type = 'methodology' - CONTEXT "Researchers want to evaluate methods first" - ORDER 1; - - -- Then primary data - NODE evidence WHERE prompt_provenance = 100 - CONTEXT "Official statistics (ONS) have highest provenance" - ORDER 2; - - -- Then peer-reviewed studies - NODE evidence WHERE prompt_publication >= 90 - CONTEXT "Academic validation matters to researchers" - ORDER 3; - - -- Then claims - NODE claims WHERE confidence_level >= 0.85 - CONTEXT "High-confidence claims after seeing evidence" - ORDER 4; - - -- Finally, counter-evidence - NODE evidence WHERE relationship_type = 'contradicts' - CONTEXT "Researchers expect to see conflicting evidence" - ORDER 5; -END -RATIONALE "Evidence-first approach for academic rigor" -CREATED_BY "journalist_jane"; -``` - -**Skeptic Path**: -```gql -CREATE NAVIGATION_PATH 'skeptic_path_inflation' -FOR INVESTIGATION 'uk_inflation_2023' -AUDIENCE 'SKEPTIC' -BEGIN - -- Start with conflicts of interest - NODE evidence ORDER BY prompt_objective DESC - CONTEXT "Skeptics want to see bias/funding disclosed first" - ORDER 1; - - -- Show counter-claims prominently - NODE claims WHERE claim_type = 'COUNTER' - CONTEXT "Skeptics expect to see dissenting views" - ORDER 2; - - -- Then methodology scrutiny - NODE evidence WHERE prompt_methodology < 80 - CONTEXT "Show evidence with weaker methodology for transparency" - ORDER 3; - - -- Then strongest evidence - NODE evidence WHERE prompt_overall > 90 - CONTEXT "Now show highest-quality evidence for balance" - ORDER 4; -END -RATIONALE "Start with skepticism-relevant dimensions" -CREATED_BY "editor_bob"; -``` - -**Affected Person Path**: -```gql -CREATE NAVIGATION_PATH 'affected_person_path_inflation' -FOR INVESTIGATION 'uk_inflation_2023' -AUDIENCE 'AFFECTED_PERSON' -BEGIN - -- Start with personal impact - NODE evidence WHERE evidence_type = 'personal_story' - CONTEXT "Affected people want to see themselves reflected" - ORDER 1; - - -- Then clear, simple data - NODE evidence WHERE title LIKE '%rent%' AND prompt_transparency > 80 - CONTEXT "Rent data is directly relevant and clearly presented" - ORDER 2; - - -- Then recommendations - NODE claims WHERE claim_type = 'RECOMMENDATION' - CONTEXT "What can I do about this?" - ORDER 3; -END -RATIONALE "Readability and personal relevance prioritized" -CREATED_BY "journalist_jane"; -``` - -### 4.3 Auto-Generated Paths - -Lith can auto-generate paths based on heuristics: - -```gql --- Auto-generate skeptic path -GENERATE NAVIGATION_PATH -FOR INVESTIGATION 'uk_inflation_2023' -AUDIENCE 'SKEPTIC' -STRATEGY BEGIN - -- 1. Find lowest objective scores (potential bias) - SELECT evidence WHERE prompt_objective < 70 LIMIT 3 - - -- 2. Find counter-claims - SELECT claims WHERE claim_type = 'COUNTER' - - -- 3. Find methodology concerns - SELECT evidence WHERE prompt_methodology < 75 LIMIT 3 - - -- 4. Balance with strongest evidence - SELECT evidence WHERE prompt_overall > 90 - - -- 5. Show consensus if exists - SELECT claims WHERE evidence_count > 5 AND confidence_level > 0.9 -END -RATIONALE "Auto-generated based on skeptic heuristics"; -``` - -### 4.4 Path Metadata as Narrative - -Paths themselves carry narrative: -```gql -INTROSPECT NAVIGATION_PATH 'researcher_path_inflation'; - --- Returns: -{ - "name": "Researcher Path: UK Inflation 2023", - "audience": "RESEARCHER", - "created_by": "journalist_jane", - "created_at": "2024-01-15T10:00:00Z", - "rationale": "Evidence-first approach for academic rigor", - "modifications": [ - { "date": "2024-01-20", "by": "editor_bob", - "change": "Added counter-evidence section", - "reason": "Peer review suggested showing dissenting views earlier" } - ], - "usage_stats": { - "views": 1542, - "avg_time_spent": "8m 32s", - "completion_rate": 0.78 - } -} -``` - -## 5. Journalism Workflows - -### 5.1 Evidence Gathering - -**Phase 1: Import from Zotero** -```gql --- Journalist uses Zotero to manage sources --- Lith imports with metadata mapping - -INSERT INTO evidence ( - title, evidence_type, url, zotero_key, - prompt_scores, added_by -) VALUES ( - 'ONS Consumer Price Inflation, UK: 2023', - 'official_statistics', - 'https://www.ons.gov.uk/cpi/2023', - 'ZOTERO_ABC123', - { provenance: 100, replicability: 100, objective: 95, - methodology: 95, publication: 100, transparency: 95 }, - 'reporter_alice' -) -RATIONALE "Official UK government statistics, gold standard for inflation data" -ZOTERO_TAGS ['inflation', 'UK', '2023', 'rent'] -IMPORTED_AT NOW(); -``` - -**Phase 2: Connect to Claims** -```gql --- Reporter creates claim -INSERT INTO claims (text, claim_type, confidence_level) -VALUES ( - 'Rent inflation (12%) exceeded overall inflation (8%) in 2023', - 'SUPPORTING', - 0.90 -) -ADDED_BY 'reporter_alice' -RATIONALE "Synthesized from ONS CPI breakdown tables"; - --- Connect evidence to claim -INSERT EDGE (claim_id, evidence_id, relationship_type, weight) -VALUES ( - 'claim_rent_inflation', - 'evidence_ons_cpi_2023', - 'SUPPORTS', - 0.95 -) -REASONING "ONS Table 3.2 shows rent component at 12.1%, headline CPI at 8.0%" -ADDED_BY 'reporter_alice' -VERIFIED_BY 'editor_bob'; -``` - -### 5.2 Collaborative Editing - -Multiple journalists work on same investigation: - -```gql --- Reporter Alice adds claim -INSERT INTO claims (...) ADDED_BY 'reporter_alice'; - --- Editor Bob requests changes -UPDATE claims -SET confidence_level = 0.85 -- was 0.90 -WHERE id = 'claim_rent_inflation' -REASON "Reduce confidence - ONS notes preliminary data subject to revision" -EDITED_BY 'editor_bob' -EDIT_TYPE 'confidence_adjustment'; - --- Reporter Alice responds -ANNOTATE claim.claim_rent_inflation -BY 'reporter_alice' -NOTE "Agreed. Final figures due March 2024, will update then."; - --- Lith journals entire conversation as narrative -``` - -### 5.3 Corrections and Retractions - -**Scenario**: ONS revises inflation figures - -```gql --- Original claim --- (created 2024-01-15) -claim: "Rent inflation reached 12% in 2023" -confidence: 0.90 -evidence: ONS preliminary data - --- ONS releases revised figures (2024-03-01) --- Actual: 12.7% - -UPDATE claims -SET text = 'Rent inflation reached 12.7% in 2023', - confidence_level = 0.95 -- Higher now that it's final data -WHERE id = 'claim_rent_inflation' -REASON "ONS released final 2023 figures on 2024-03-01. - Preliminary data (12%) underestimated actual (12.7%). - Confidence increased as this is now final, not preliminary." -CORRECTION_TYPE 'factual_update' -DISCLOSED_AT NOW() -DISCLOSED_BY 'reporter_alice' -SOURCE_URL 'https://ons.gov.uk/final-2023-cpi'; - --- Lith preserves original in journal --- Readers see correction notice with full context -``` - -**Retraction** (more serious): -```gql -UPDATE claims -SET confidence_level = 0.0, - retracted = TRUE, - retracted_at = NOW() -WHERE id = 'claim_controversial' -REASON "Primary source retracted statement. Original interview audio - revealed misquote. See correction notice published 2024-02-15." -RETRACTION_TYPE 'source_error' -DISCLOSED_BY 'editor_bob' -APOLOGY "We apologize for the error and have updated our editorial - processes to prevent similar mistakes."; - --- Original claim remains queryable but marked retracted --- All dependent claims are flagged for review -``` - -### 5.4 Fact-Checking Workflow - -```gql --- Fact-checker reviews claim -SELECT claim.text, - evidence.title, - evidence.prompt_overall, - relationship.weight -FROM claims - JOIN relationships ON claims.id = relationships.from_id - JOIN evidence ON relationships.to_id = evidence.id -WHERE claims.id = 'claim_to_check' -ORDER BY evidence.prompt_overall DESC; - --- Fact-checker adds verification note -ANNOTATE claim.claim_to_check -BY 'fact_checker_charlie' -VERIFICATION_STATUS 'verified' -NOTE "Cross-checked with 3 independent sources. ONS data confirmed. - Academic study methodology sound (peer-reviewed, n=5000). - Think tank report methodology less rigorous but directionally correct." -VERIFIED_AT NOW() -PROMPT_ADJUSTMENTS { - evidence_ons_cpi: "No changes, score accurate", - evidence_academic_study: "Increased objective score from 75→80 after bias review", - evidence_think_tank: "No changes" -}; - --- Lith journals verification as part of narrative -``` - -## 6. BoFIG Integration: Case Study - -### 6.1 BoFIG Architecture - -**BoFIG** (Binary-Origami Figuration) is an epistemic infrastructure system built on Lith. - -``` -┌─────────────────────────────────────────────┐ -│ BoFIG Frontend (Phoenix LiveView) │ -│ - Graph visualization (D3.js) │ -│ - PROMPT scoring interface │ -│ - Navigation path editor │ -└────────────────┬────────────────────────────┘ - │ GraphQL API (Absinthe) -┌────────────────▼────────────────────────────┐ -│ BoFIG Business Logic (Elixir) │ -│ - Claims context │ -│ - Evidence context │ -│ - Relationships context │ -│ - Navigation paths │ -└────────────────┬────────────────────────────┘ - │ GQL Queries -┌────────────────▼────────────────────────────┐ -│ Lith (Forth/Zig/Factor) │ -│ - Narrative-first database │ -│ - Document + edge collections │ -│ - PROMPT scores as metadata │ -│ - Provenance tracking │ -│ - Reversible operations │ -└─────────────────────────────────────────────┘ -``` - -### 6.2 Schema Mapping - -**BoFIG Collections in Lith**: - -```gql --- Claims (investigative journalism claims) -CREATE COLLECTION bofig_claims ( - id UUID PRIMARY KEY, - investigation_id VARCHAR NOT NULL, - text TEXT NOT NULL, - claim_type ENUM('PRIMARY', 'SUPPORTING', 'COUNTER', 'RECOMMENDATION'), - confidence_level FLOAT CHECK (confidence_level BETWEEN 0.0 AND 1.0), - added_by VARCHAR, - added_at TIMESTAMP, - last_verified TIMESTAMP, - verified_by VARCHAR -) WITH NARRATIVE_METADATA; - --- Evidence (sources with PROMPT scores) -CREATE COLLECTION bofig_evidence ( - id UUID PRIMARY KEY, - investigation_id VARCHAR NOT NULL, - title VARCHAR NOT NULL, - evidence_type ENUM('official_statistics', 'academic_study', 'think_tank', - 'expert_interview', 'personal_story', 'methodology'), - url TEXT, - zotero_key VARCHAR UNIQUE, - prompt_scores STRUCT { - provenance INT, - replicability INT, - objective INT, - methodology INT, - publication INT, - transparency INT, - overall COMPUTED AS (provenance + replicability + objective + - methodology + publication + transparency) / 6.0 - }, - added_by VARCHAR, - added_at TIMESTAMP -) WITH NARRATIVE_METADATA; - --- Relationships (claim ←→ evidence graph edges) -CREATE EDGE_COLLECTION bofig_relationships ( - from_id UUID NOT NULL, - to_id UUID NOT NULL, - relationship_type ENUM('SUPPORTS', 'CONTRADICTS', 'CONTEXTUALIZES'), - weight FLOAT CHECK (weight BETWEEN 0.0 AND 1.0), - confidence FLOAT CHECK (confidence BETWEEN 0.0 AND 1.0), - reasoning TEXT NOT NULL, - added_by VARCHAR, - added_at TIMESTAMP, - verified_by VARCHAR -) WITH NARRATIVE_METADATA; - --- Navigation Paths (boundary objects) -CREATE COLLECTION bofig_navigation_paths ( - id UUID PRIMARY KEY, - investigation_id VARCHAR NOT NULL, - name VARCHAR NOT NULL, - audience_type ENUM('RESEARCHER', 'POLICYMAKER', 'SKEPTIC', 'AFFECTED_PERSON'), - description TEXT, - path_nodes ARRAY OF STRUCT { - entity_id UUID, - entity_type ENUM('CLAIM', 'EVIDENCE'), - order INT, - context TEXT - }, - created_by VARCHAR, - created_at TIMESTAMP -) WITH NARRATIVE_METADATA; -``` - -### 6.3 Query Examples - -**GQL queries replacing AQL**: - -```gql --- Get all high-confidence claims with supporting evidence -SELECT claims.text, claims.confidence_level, - ARRAY_AGG(evidence.title ORDER BY evidence.prompt_overall DESC) AS evidence_titles, - AVG(evidence.prompt_overall) AS avg_evidence_quality -FROM bofig_claims AS claims - JOIN bofig_relationships AS rel ON claims.id = rel.from_id - JOIN bofig_evidence AS evidence ON rel.to_id = evidence.id -WHERE claims.investigation_id = 'uk_inflation_2023' - AND claims.confidence_level >= 0.80 - AND rel.relationship_type = 'SUPPORTS' -GROUP BY claims.id -ORDER BY claims.confidence_level DESC, avg_evidence_quality DESC; - --- Get evidence chain (graph traversal) -TRAVERSE FROM claim_id - FOLLOW bofig_relationships OUTBOUND - MAX_DEPTH 3 - FILTER relationship_type IN ('SUPPORTS', 'CONTEXTUALIZES') - RETURN { - path: path.entities, - total_weight: SUM(path.relationships.weight), - avg_prompt: AVG(path.entities[type='EVIDENCE'].prompt_overall) - } -[PROVENANCE]; - --- Get navigation path for skeptical audience -SELECT path.name, path.description, path.path_nodes -FROM bofig_navigation_paths AS path -WHERE path.investigation_id = 'uk_inflation_2023' - AND path.audience_type = 'SKEPTIC' -ORDER BY path.created_at DESC -LIMIT 1; -``` - -### 6.4 Migration Benefits - -**Before** (ArangoDB): -- ❌ No provenance tracking (who added this evidence? when? why?) -- ❌ No correction workflow (retractions require manual tracking) -- ❌ PROMPT scores are static (no history of changes) -- ❌ Relationships lack narrative (why does this edge exist?) -- ❌ No audit trail (who verified this claim?) - -**After** (Lith): -- ✅ Full provenance: Every entity knows its origin -- ✅ Reversible corrections: Update with reason, preserve original -- ✅ PROMPT score evolution: Track score changes over time -- ✅ Narrative relationships: Every edge has rationale -- ✅ Complete audit trail: All operations journaled - -## 7. My-Newsroom Integration - -### 7.1 Multi-Agent Verification - -My-Newsroom uses Dempster-Shafer belief fusion (50-100 agents) for claim verification. Lith provides the audit layer. - -**Architecture**: -``` -┌──────────────────────────────────────────────┐ -│ My-Newsroom (Elixir/OTP Ensemble) │ -│ - 50-100 specialized agents │ -│ - Dempster-Shafer belief fusion (Julia) │ -│ - Byzantine fault tolerance │ -└────────────────┬─────────────────────────────┘ - │ Audit Trail -┌────────────────▼─────────────────────────────┐ -│ Lith Epistemic Ledger │ -│ - Every belief fusion recorded │ -│ - Agent reasoning preserved │ -│ - Conflicts/resolutions journaled │ -└──────────────────────────────────────────────┘ -``` - -### 7.2 Belief Fusion Audit Trail - -```gql -CREATE COLLECTION newsroom_belief_fusions ( - fusion_id UUID PRIMARY KEY, - claim_text TEXT NOT NULL, - agent_pool ARRAY OF VARCHAR, -- ["agent_reporter_politics", "agent_fact_checker", ...] - individual_beliefs ARRAY OF STRUCT { - agent_id VARCHAR, - belief FLOAT, -- 0.0-1.0 - mass_function JSONB, -- Dempster-Shafer mass function - rationale TEXT, - sources_cited ARRAY OF VARCHAR - }, - fusion_method ENUM('Dempster', 'Yager', 'DuboisPrade', 'Average'), - fusion_result STRUCT { - fused_belief FLOAT, - fused_mass JSONB, - conflict_measure FLOAT, - confidence_interval STRUCT {lower FLOAT, upper FLOAT} - }, - consensus_reached BOOLEAN, - consensus_threshold FLOAT, - fused_by VARCHAR, -- Orchestrator agent - fused_at TIMESTAMP -) WITH NARRATIVE_METADATA; - --- Example: Claim verification -INSERT INTO newsroom_belief_fusions ( - claim_text, agent_pool, individual_beliefs, - fusion_method, fusion_result, consensus_reached -) VALUES ( - 'UK rent inflation exceeded 12% in 2023', - ['agent_reporter_economics', 'agent_fact_checker_primary', 'agent_editor_senior'], - [ - { agent_id: 'agent_reporter_economics', - belief: 0.90, - mass_function: '{"{{true}}": 0.90, "Θ": 0.10}', - rationale: 'ONS data Table 3.2 shows 12.1% rent component', - sources_cited: ['evidence_ons_cpi_2023'] }, - { agent_id: 'agent_fact_checker_primary', - belief: 0.85, - mass_function: '{"{{true}}": 0.85, "Θ": 0.15}', - rationale: 'Cross-checked with 3 sources, slight variations (11.8-12.3%)', - sources_cited: ['evidence_ons_cpi_2023', 'evidence_academic_study', 'evidence_think_tank'] }, - { agent_id: 'agent_editor_senior', - belief: 0.95, - mass_function: '{"{{true}}": 0.95, "Θ": 0.05}', - rationale: 'Final ONS figures released, no longer preliminary', - sources_cited: ['evidence_ons_final_2023'] } - ], - 'Dempster', - { - fused_belief: 0.92, - fused_mass: '{"{{true}}": 0.92, "Θ": 0.08}', - conflict_measure: 0.03, - confidence_interval: {lower: 0.88, upper: 0.96} - }, - TRUE -- Consensus reached -) -FUSED_BY 'orchestrator_agent_main' -RATIONALE "Three agents agree with high confidence. Low conflict (0.03). - Consensus threshold (0.85) exceeded. Claim verified." -[PROVENANCE]; -``` - -### 7.3 Agent Introspection - -Agents query their own reasoning history: - -```gql --- Agent asks: "What claims have I verified about inflation?" -SELECT fusion.claim_text, - belief.belief, - belief.rationale, - fusion.fusion_result.fused_belief, - fusion.consensus_reached -FROM newsroom_belief_fusions AS fusion, - UNNEST(fusion.individual_beliefs) AS belief -WHERE belief.agent_id = 'agent_reporter_economics' - AND fusion.claim_text LIKE '%inflation%' -ORDER BY fusion.fused_at DESC -[PROVENANCE]; - --- Returns: --- [ --- { claim: "UK rent inflation exceeded 12% in 2023", --- my_belief: 0.90, --- my_rationale: "ONS data Table 3.2 shows 12.1% rent component", --- consensus: 0.92, --- consensus_reached: true, --- timestamp: "2024-01-15T14:30:00Z" }, --- ... --- ] -``` - -### 7.4 Conflict Resolution Narrative - -When agents disagree: - -```gql --- Agents disagree on controversial claim -INSERT INTO newsroom_belief_fusions ( - claim_text, individual_beliefs, fusion_method, fusion_result, consensus_reached -) VALUES ( - 'Government inflation target was achievable in 2023', - [ - { agent: 'agent_economist_keynesian', belief: 0.30, - rationale: 'Structural factors made 2% target unrealistic' }, - { agent: 'agent_economist_monetarist', belief: 0.75, - rationale: 'BoE had tools but lacked political will' }, - { agent: 'agent_policy_analyst', belief: 0.50, - rationale: 'Uncertain - depends on counterfactuals' } - ], - 'DuboisPrade', -- Use DuboisPrade for high-conflict scenarios - { - fused_belief: 0.52, - conflict_measure: 0.45, -- HIGH CONFLICT - confidence_interval: {lower: 0.30, upper: 0.75} - }, - FALSE -- No consensus -) -RATIONALE "High conflict (0.45) due to ideological differences. - DuboisPrade fusion used (handles conflict better than Dempster). - No consensus reached - mark claim as CONTESTED. - Editor review required." -REQUIRES_HUMAN_REVIEW TRUE -[PROVENANCE]; - --- Lith journals this as a contested claim --- Readers see: "This claim is disputed among experts (consensus: 52%, conflict: high)" -``` - -## 8. Implementation Roadmap - -### Phase 1: Lith Core for BoFIG (Month 1-3) - -- [ ] Migrate BoFIG from ArangoDB to Lith -- [ ] Implement PROMPT score schema -- [ ] Build navigation path collections -- [ ] Create GQL equivalents for all current AQL queries -- [ ] Migrate UK Inflation 2023 test dataset -- [ ] Test with NUJ journalists (25 users) - -### Phase 2: Zotero Integration (Month 4-6) - -- [ ] Formalize Zotero metadata → Lith mapping -- [ ] Build browser extension (import from Zotero to Lith) -- [ ] Implement two-way sync (changes in Lith → Zotero) -- [ ] PROMPT score estimation from Zotero tags -- [ ] Citation graph import (related works) - -### Phase 3: My-Newsroom Integration (Month 7-12) - -- [ ] Define belief fusion audit schema -- [ ] Build Elixir adapter (My-Newsroom → Lith) -- [ ] Implement agent introspection queries -- [ ] Build conflict resolution workflow -- [ ] Test with 10-agent newsroom (proof-of-concept) - -### Phase 4: i-doc Platform (Month 13-18) - -- [ ] D3.js visualization (evidence graphs) -- [ ] LiveView UI (interactive navigation) -- [ ] Auto-generate navigation paths -- [ ] Embed PROMPT scoring interface -- [ ] Public-facing i-doc viewer - -### Phase 5: Scale to Reuters-level Newsroom (Month 19-24) - -- [ ] 50-100 agent deployment (My-Newsroom full scale) -- [ ] Byzantine fault tolerance testing -- [ ] Distributed Lith (Raft consensus, sharding) -- [ ] Real-time collaboration (Phoenix Channels) -- [ ] IPFS provenance integration - -## 9. Evaluation Metrics - -### 9.1 Journalism Quality - -- **Correction Rate**: % of claims corrected within 30 days - - **Target**: <5% (high initial accuracy) - - **Metric**: `SELECT COUNT(*) FROM claims WHERE corrected = TRUE / COUNT(*)` - -- **Retraction Rate**: % of claims fully retracted - - **Target**: <1% (rare, serious errors) - - **Metric**: `SELECT COUNT(*) FROM claims WHERE retracted = TRUE / COUNT(*)` - -- **Average PROMPT Score**: Quality of evidence base - - **Target**: >75 (acceptable quality) - - **Metric**: `SELECT AVG(prompt_overall) FROM evidence` - -- **Verification Latency**: Time from claim to verification - - **Target**: <48 hours - - **Metric**: `SELECT AVG(verified_at - added_at) FROM claims WHERE verified_by IS NOT NULL` - -### 9.2 Reader Engagement - -- **Navigation Path Completion**: % of readers who complete path - - **Target**: >60% - - **Tracked via**: Frontend analytics + Lith queries - -- **Average Time on Evidence**: Reader engagement depth - - **Target**: >3 minutes per evidence item - - **Tracked via**: Frontend analytics - -- **Perspective Diversity**: % readers exploring multiple paths - - **Target**: >30% try 2+ audience paths - - **Tracked via**: Session tracking - -### 9.3 Agent Performance - -- **Consensus Rate**: % of claims reaching agent consensus - - **Target**: >80% (most claims should converge) - - **Metric**: `SELECT COUNT(*) WHERE consensus_reached = TRUE / COUNT(*)` - -- **Conflict Detection**: % of genuine conflicts identified - - **Target**: >90% (high sensitivity) - - **Metric**: Manual review of high-conflict fusions - -- **Byzantine Resilience**: % of attacks detected/prevented - - **Target**: 100% up to 33% malicious agents - - **Tested via**: Adversarial agent injection - -## 10. Related Work - -### 10.1 i-doc Platforms - -- **Korsakow** (Florian Thalhofer): Interactive documentary authoring - - **Gap**: No provenance tracking, no PROMPT scores -- **Zeega** (Zeega Project): Multi-layered storytelling - - **Gap**: No epistemological metadata -- **Eko** (Formerly Interlude): Interactive video platform - - **Gap**: Entertainment-focused, not journalism - -**Lith Advantage**: Purpose-built for journalism epistemology. - -### 10.2 Fact-Checking Systems - -- **ClaimBuster** (UTA): Automated claim detection - - **Gap**: No narrative database, no evidence tracking -- **Full Fact** (UK): Manual fact-checking - - **Gap**: Uses traditional databases, no provenance -- **PolitiFact** (Poynter): Truth-O-Meter ratings - - **Gap**: Ratings are final, no correction workflow - -**Lith Advantage**: Treats corrections as first-class operations. - -### 10.3 Evidence Management - -- **Zotero**: Reference management - - **Integration**: Lith imports from Zotero -- **ResearchRabbit**: Citation graph visualization - - **Gap**: No epistemological scoring -- **Scite**: Citation context (supporting/contrasting) - - **Inspiration**: Similar to Lith's relationship types - -**Lith Advantage**: Combines evidence management + epistemology + narrative. - -## 11. Open Questions - -See `lith.scm` Q-IDOC-* questions: - -1. **Q-IDOC-PROMPT-001**: Should PROMPT scores be normalized across investigations? - - **Issue**: Different domains have different score distributions - - **Proposal**: Domain-specific normalization curves - -2. **Q-IDOC-PATH-001**: How to measure navigation path effectiveness? - - **Metrics**: Completion rate, time spent, user satisfaction - - **Proposal**: A/B testing of auto-generated vs. manual paths - -3. **Q-IDOC-CORRECTION-001**: When should corrections trigger re-verification? - - **Threshold**: Major corrections (>10% confidence change)? - - **Proposal**: Cascading re-verification for dependent claims - -4. **Q-IDOC-AGENT-001**: How many agents needed for reliable consensus? - - **Minimum**: 3 (for voting) - - **Optimal**: 7-15 (diminishing returns beyond this) - - **Research needed**: Empirical studies - -## 12. Conclusion - -Lith is not just a database for journalism—it is **infrastructure for epistemology in the post-truth era**. By making provenance, corrections, and multi-perspective navigation first-class database semantics, Lith enables: - -1. **i-docs at scale**: Navigation over narration, reader agency -2. **Transparent epistemology**: PROMPT scores, evidence chains -3. **Correction workflows**: Retractions with full context -4. **Multi-agent verification**: Dempster-Shafer fusion audit trails -5. **Boundary objects**: Same evidence, multiple audience perspectives - -BoFIG demonstrates Lith's practical application in investigative journalism. My-Newsroom extends this to multi-agent verification. Together, they form a complete epistemic infrastructure stack for journalism in the 21st century. - -**Next Steps**: -1. Migrate BoFIG from ArangoDB to Lith (Month 1-3) -2. User testing with NUJ journalists (Month 3, 6, 12) -3. My-Newsroom integration (Month 7-12) -4. Full-scale Reuters-level newsroom (Month 19-24) - -**Impact**: Lith + BoFIG + My-Newsroom could become the **de facto standard** for i-doc journalism, much as Zotero became the standard for academic reference management. - ---- - -**Document Status**: Living document. Updates tracked in `spec/WP05_CHANGELOG.md`. - -**See Also**: -- [BoFIG Repository](https://github.com/Hyperpolymath/bofig) -- [My-Newsroom Repository](https://github.com/hyperpolymath/my-newsroom) -- [MIT Open Doc Lab](https://opendoclab.mit.edu/) -- [PROMPT Framework](https://example.com/prompt-framework) diff --git a/lithoglyph/docs/gql-safety-model.adoc b/lithoglyph/docs/gql-safety-model.adoc deleted file mode 100644 index 25afcaa6..00000000 --- a/lithoglyph/docs/gql-safety-model.adoc +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) - -= GQL Safety Model — Two-Tier Architecture -:toc: left -:toclevels: 3 -:sectnums: - -== Overview - -GQL (Graph Query Language) follows the same two-tier model as VCL: - -[cols="1,2,3", options="header"] -|=== -| Tier | What | How - -| **GQL** -| The query language -| Graph queries with dependent types (`CREATE COLLECTION ... WITH DEPENDENT_TYPES`). - What users write. - -| **GQL Safety Pipeline** -| Progressive type safety -| Applies TypeLL's 10 levels behind GQL automatically. Simple queries exit early. - Proof-carrying queries activate L9-L10. -|=== - -== Relationship to VCL-UT - -The 10 safety levels originate in **TypeLL** (the core type theory). VCL-UT is -the reference implementation for database queries. GQL maps its own constructs -to the same 10 levels — it does not have a separate "-UT" repo. - -[source] ----- -TypeLL (type theory core — defines the 10 levels) - │ - ├──→ VCL-UT (reference implementation, VeriSimDB) - ├──→ KQL (applies same levels, QuandleDB) - └──→ GQL (applies same levels, LithoGlyph) ----- - -== GQL Safety Levels - -[cols="^1,<3,<5", options="header"] -|=== -| Level | Name | GQL Mapping - -| L1 | Construction Safety | Query injection prevention (safe graph traversal parsing) -| L2 | Schema Pinning | Collection schema checked with dependent types (`BoundedNat`, `NonEmptyString`) -| L3 | Resource Linearity | Graph traversal cursor/connection leak prevention -| L4 | Session Protocols | Query session state machine (connect → traverse → close) -| L5 | Effect Tracking | Side-effect auditing for graph mutations -| L6 | Scope Isolation | Collection namespace boundaries -| L7 | Information Flow | Graph data lineage and provenance tracking -| L8 | Quantitative Bounds | Bounded traversal depth (prevent infinite graph walks) -| L9 | Proof Attachment | Dependent type proofs on collection schemas -| L10 | Cross-Cutting | Composition safety across graph joins and federation -|=== - -== GQL-Specific Type Features - -GQL has dependent types built into its collection definitions: - -[cols="<3,^1,<4", options="header"] -|=== -| Feature | Level | Description - -| Bounded Numeric Types -| L2, L8 -| `BoundedNat min max` — numeric fields with compile-time range proofs. - Prevents out-of-range values at the type level. - -| Required/Optional Strings -| L2 -| `NonEmptyString` vs `Option String` — required fields cannot be null/empty - at the type level. - -| Confidence Types -| L9 -| `Confidence` type for prompt-scored fields with dependent bounds. - -| Provenance Tracking -| L7 -| `WITH PROVENANCE_TRACKING` on collections — graph lineage at the schema level. - -| Dependent Collection Schemas -| L2, L9 -| `CREATE COLLECTION ... WITH DEPENDENT_TYPES` — schema fields carry type-level - constraints verified by Idris2 ABI (`src/Lith/`). -|=== - -== GQL-DT Preview - -The LithoGlyph Studio includes a GQL-DT preview panel (`studio/lib/bs/src/GqldtPreview.res`) -that generates dependent-type-annotated GQL from collection definitions. This component -visualises how L2 and L9 safety levels apply to a given schema. - -=== Historical Note - -"GQL-DT" was the original name for GQL's dependent type features. It is not a separate -product — it describes how GQL uses dependent types within the safety pipeline, primarily -at levels L2 and L9. The `GqldtPreview` component name is retained for compatibility but -represents "GQL safety preview" rather than a separate execution mode. - -== Implementation Status - -[cols="^1,<3,^1", options="header"] -|=== -| Level | Status | Notes - -| L1-L2 | Implemented | Collection schemas with dependent types (ReScript + Factor + Zig) -| L3-L4 | Designed | Multi-backend connections (Factor core, Forth core, Zig core) -| L5-L7 | Partial | Provenance tracking specified, mutations audited -| L8 | Designed | Bounded traversal depth for graph walks -| L9 | Partial | GqldtPreview generates dependent type annotations -| L10 | Designed | Cross-collection composition safety -|=== - -== Implementation Stack - -LithoGlyph has three core implementations: - -[cols="<2,<2,<4", options="header"] -|=== -| Core | Language | Purpose - -| `core-factor/` | Factor | GQL engine with C API bridge (`gql-c-api/`) -| `core-forth/` | Forth | Minimal GQL engine -| `core-zig/` | Zig | FFI-compatible GQL engine (Idris2 ABI at `src/Lith/`) -|=== - -The Idris2 ABI definitions in `src/Lith/` (LithBridge, LithForeign, LithLayout) provide -the formal type proofs that underpin levels L2 and L9. - -== Reference - -* `studio/lib/bs/src/GqldtPreview.res` — GQL dependent type preview component -* `src/Lith/` — Idris2 ABI definitions (LithBridge, LithForeign, LithLayout) -* `ARCHITECTURE.adoc` — Full system architecture -* https://github.com/hyperpolymath/vcl-ut[VCL-UT] — Reference implementation of the 10-level safety pipeline diff --git a/lithoglyph/eclexiaiser.toml b/lithoglyph/eclexiaiser.toml deleted file mode 100644 index b24ee6a0..00000000 --- a/lithoglyph/eclexiaiser.toml +++ /dev/null @@ -1,26 +0,0 @@ -# eclexiaiser manifest — energy/carbon resource budgets -# SPDX-License-Identifier: MPL-2.0 - -[project] -name = "lithoglyph" - -[[functions]] -name = "encodeProvenance" -source = "core-zig/src/cbor.zig" -energy-budget-mj = 30.0 -carbon-budget-mg = 6.0 - -[[functions]] -name = "crc32c" -source = "core-zig/src/blocks.zig" -energy-budget-mj = 10.0 -carbon-budget-mg = 2.0 - -[carbon] -provider = "static" -region = "GB" -static-intensity = 200.0 - -[report] -format = "text" -include-recommendations = true diff --git a/lithoglyph/examples/SafeDOMExample.affine b/lithoglyph/examples/SafeDOMExample.affine deleted file mode 100644 index 2a62c1d1..00000000 --- a/lithoglyph/examples/SafeDOMExample.affine +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SafeDOMExample.affine — formally-verified DOM mounting (aspirational). -// -// This example shows the *shape* of SafeDOM consumer code in current -// AffineScript syntax. The `SafeDOM` stdlib surface it references -// (`mount_safe`, `mount_when_ready`, `mount_batch`, -// `proven_selector_validate`, `proven_html_validate`, `mount`) is the -// target of `affinescript#56` (DOM+Pixi binding survey) and does not -// yet exist in the published stdlib. The file is therefore -// parse-checked but not type-checked end-to-end until #56 lands the -// bindings; `affinescript check` reports `Resolve.UndefinedModule -// SafeDOM` which is expected. -// -// Previous versions of this file (estate-wide, 5 dialect variants) -// pre-dated ADR-014 (qualified paths), ADR-016 (effect rows), and the -// `#{`-record-literal sigil (ADR-215). They were retired in favour of -// this canonical via the gitbot-fleet#208 sweep (2026-05-26). - -module SafeDOMExample; - -use prelude::{Option, Some, None, Result, Ok, Err}; - -// `Element` and friends are nominal extern types for now — the real -// shape lands with affinescript#56. -extern type Element; -extern type Selector; -extern type ValidHTML; - -// Single-mount status, lifted from the host into a typed tag union. -enum MountStatus { - Mounted(Element), - MountPointNotFound(String), - InvalidSelector(String), - InvalidHTML(String) -} - -// Batch-mount result. -enum MountResult { - Mounted([Element]), - Failed(String) -} - -// Spec for one element in a batch mount. -struct MountSpec { - selector: String, - html: String -} - -// SafeDOM's host-side surface, all IO-effecting. Callbacks are passed -// as separate parameters (rather than a `MountCallbacks` record) -// because fn-typed struct fields are not currently parser-supported. -extern fn mount_safe( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_when_ready( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_batch(specs: ref [MountSpec]) -{IO}-> MountResult; - -extern fn proven_selector_validate(s: ref String) -{IO}-> Result; -extern fn proven_html_validate(s: ref String) -{IO}-> Result; -extern fn mount(sel: ref Selector, html: ref ValidHTML) -{IO}-> MountStatus; - -extern fn array_for_each(xs: ref [Element], f: fn(Element) -> ()) -{IO}-> (); -extern fn array_len(xs: ref [Element]) -> Int; - -// Example 1 — basic mount with success/error branches. -pub fn mount_app() -{IO}-> () { - mount_safe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - fn(el) -> () { Console::log("App mounted successfully"); }, - fn(err) -> () { Console::error("Mount failed: " ++ err); }, - ); -} - -// Example 2 — defer until DOM ready. -pub fn mount_when_dom_ready() -{IO}-> () { - mount_when_ready( - "#app", - "

App Title

", - fn(_el) -> () { Console::log("Mounted after DOM ready"); }, - fn(err) -> () { Console::error("Failed: " ++ err); }, - ); -} - -// Example 3 — atomic batch mount. -pub fn mount_multiple() -{IO}-> () { - let specs = [ - MountSpec #{ selector: "#header", html: "

Site Title

" }, - MountSpec #{ selector: "#nav", html: "" }, - MountSpec #{ selector: "#main", html: "

Content here

" }, - MountSpec #{ selector: "#footer", html: "
2026
" }, - ]; - - match mount_batch(specs) { - Mounted(elements) => { - Console::log("Batch mount succeeded"); - array_for_each(elements, fn(_el) -> () { Console::log(" element"); }); - }, - Failed(err) => { - Console::error("Batch mount failed (atomic — none mounted): " ++ err); - } - } -} - -// Example 4 — explicit two-stage validation before mounting. -pub fn mount_with_validation() -{IO}-> () { - match proven_selector_validate("#my-app") { - Err(e) => Console::error("Invalid selector: " ++ e), - Ok(valid_selector) => match proven_html_validate("
Content
") { - Err(e) => Console::error("Invalid HTML: " ++ e), - Ok(valid_html) => match mount(valid_selector, valid_html) { - Mounted(_el) => Console::log("Mounted with validated inputs"), - MountPointNotFound(s) => Console::error("Element not found: " ++ s), - InvalidSelector(_) => Console::error("impossible — already validated"), - InvalidHTML(_) => Console::error("impossible — already validated"), - }, - }, - } -} diff --git a/lithoglyph/examples/demo-server.zig b/lithoglyph/examples/demo-server.zig deleted file mode 100644 index 225133a6..00000000 --- a/lithoglyph/examples/demo-server.zig +++ /dev/null @@ -1,295 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph Phase 4 Demo - Minimal HTTP server demonstrating complete stack -// -// Architecture demonstrated: -// HTTP Request → Zig Server → Bridge FFI → BlockStorage → Persistent .lgh files - -const std = @import("std"); - -// FFI types matching our Phase 3 bindings -const LgBlob = extern struct { - ptr: ?[*]const u8, - len: usize, - - fn empty() LgBlob { - return .{ .ptr = null, .len = 0 }; - } - - fn toSlice(self: LgBlob) ?[]const u8 { - if (self.ptr) |p| return p[0..self.len]; - return null; - } -}; - -const LgStatus = enum(c_int) { - ok = 0, - err_internal = 1, - err_not_found = 2, - err_invalid_argument = 3, - err_out_of_memory = 4, - err_not_implemented = 5, - err_txn_not_active = 6, - err_txn_already_committed = 7, -}; - -const LgResult = extern struct { - value: LgBlob, - err: LgBlob, - status: LgStatus, -}; - -// External FFI functions (from libbridge.so) -extern fn lith_version() c_int; -extern fn lith_db_open( - path: [*]const u8, - path_len: usize, - opts: ?[*]const u8, - opts_len: usize, - out_db: *?*anyopaque, - out_err: *LgBlob, -) LgStatus; -extern fn lith_db_close(db: ?*anyopaque) void; -extern fn lith_txn_begin( - db: ?*anyopaque, - read_only: bool, - out_txn: *?*anyopaque, - out_err: *LgBlob, -) LgStatus; -extern fn lith_txn_commit(txn: ?*anyopaque, out_err: *LgBlob) LgStatus; -extern fn lith_apply(txn: ?*anyopaque, op: [*]const u8, op_len: usize) LgResult; -extern fn lith_introspect_schema( - db: ?*anyopaque, - out_schema: *LgBlob, - out_err: *LgBlob, -) LgStatus; -extern fn lith_blob_free(blob: *LgBlob) void; - -// Global state -var db: ?*anyopaque = null; -var gpa: std.heap.GeneralPurposeAllocator(.{}) = undefined; - -pub fn main() !void { - gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - std.debug.print("===========================================\n", .{}); - std.debug.print("Lithoglyph Phase 4 Demo Server\n", .{}); - std.debug.print("===========================================\n", .{}); - std.debug.print("Bridge version: {d}\n", .{lith_version()}); - - // Open database - const db_path = "demo.lgh"; - var err_blob = LgBlob.empty(); - - const status = lith_db_open( - db_path.ptr, - db_path.len, - null, - 0, - &db, - &err_blob, - ); - - if (status != .ok) { - if (err_blob.toSlice()) |err| { - std.debug.print("Failed to open database: {s}\n", .{err}); - } - return error.DatabaseOpenFailed; - } - - std.debug.print("Database opened: {s}\n", .{db_path}); - std.debug.print("DB handle: {*}\n\n", .{db}); - - defer { - if (db) |d| { - lith_db_close(d); - std.debug.print("\nDatabase closed\n", .{}); - } - } - - // Simple HTTP server - const address = try std.net.Address.parseIp("127.0.0.1", 8080); - var listener = try address.listen(.{ - .reuse_address = true, - }); - defer listener.deinit(); - - std.debug.print("Server listening on http://127.0.0.1:8080\n", .{}); - std.debug.print("\nEndpoints:\n", .{}); - std.debug.print(" GET /health - Health check\n", .{}); - std.debug.print(" GET /version - Bridge version\n", .{}); - std.debug.print(" POST /insert - Insert document\n", .{}); - std.debug.print(" GET /schema - Introspect schema\n", .{}); - std.debug.print("\n===========================================\n\n", .{}); - - // Accept connections - while (true) { - const conn = try listener.accept(); - _ = try std.Thread.spawn(.{}, handleConnection, .{ allocator, conn }); - } -} - -fn handleConnection(allocator: std.mem.Allocator, conn: std.net.Server.Connection) void { - defer conn.stream.close(); - - var buf: [4096]u8 = undefined; - const bytes_read = conn.stream.read(&buf) catch return; - if (bytes_read == 0) return; - - const request = buf[0..bytes_read]; - - // Parse HTTP request line - var lines = std.mem.splitScalar(u8, request, '\n'); - const first_line = lines.next() orelse return; - - var parts = std.mem.splitScalar(u8, first_line, ' '); - const method = parts.next() orelse return; - const path = parts.next() orelse return; - - std.debug.print("[{s}] {s}\n", .{ method, path }); - - // Route requests - if (std.mem.eql(u8, path, "/health")) { - handleHealth(conn.stream) catch {}; - } else if (std.mem.eql(u8, path, "/version")) { - handleVersion(conn.stream) catch {}; - } else if (std.mem.eql(u8, path, "/insert")) { - handleInsert(allocator, conn.stream, request) catch {}; - } else if (std.mem.eql(u8, path, "/schema")) { - handleSchema(conn.stream) catch {}; - } else { - send404(conn.stream) catch {}; - } -} - -fn handleHealth(stream: std.net.Stream) !void { - const response = - \\HTTP/1.1 200 OK - \\Content-Type: application/json - \\Connection: close - \\ - \\{"status":"healthy","database":"open","bridge_version":100} - ; - _ = try stream.writeAll(response); -} - -fn handleVersion(stream: std.net.Stream) !void { - var buf: [256]u8 = undefined; - const version = lith_version(); - const json = try std.fmt.bufPrint(&buf, - \\HTTP/1.1 200 OK - \\Content-Type: application/json - \\Connection: close - \\ - \\{{"version":{d}}} - , .{version}); - _ = try stream.writeAll(json); -} - -fn handleInsert(allocator: std.mem.Allocator, stream: std.net.Stream, request: []const u8) !void { - _ = allocator; - - // Begin transaction - var txn: ?*anyopaque = null; - var err_blob = LgBlob.empty(); - - var status = lith_txn_begin(db, false, &txn, &err_blob); - if (status != .ok) { - const error_response = - \\HTTP/1.1 500 Internal Server Error - \\Content-Type: application/json - \\Connection: close - \\ - \\{"error":"transaction_failed"} - ; - _ = try stream.writeAll(error_response); - return; - } - - // Extract body (simplified - just use placeholder) - _ = request; - const op = "{\"op\":\"insert\",\"collection\":\"demo\",\"doc\":{\"name\":\"test\"}}"; - - // Apply operation - const result = lith_apply(txn, op.ptr, op.len); - - if (result.status != .ok) { - const error_response = - \\HTTP/1.1 500 Internal Server Error - \\Content-Type: application/json - \\Connection: close - \\ - \\{"error":"apply_failed"} - ; - _ = try stream.writeAll(error_response); - return; - } - - // Commit - status = lith_txn_commit(txn, &err_blob); - if (status != .ok) { - const error_response = - \\HTTP/1.1 500 Internal Server Error - \\Content-Type: application/json - \\Connection: close - \\ - \\{"error":"commit_failed"} - ; - _ = try stream.writeAll(error_response); - return; - } - - const success_response = - \\HTTP/1.1 200 OK - \\Content-Type: application/json - \\Connection: close - \\ - \\{"status":"ok","inserted":true} - ; - _ = try stream.writeAll(success_response); -} - -fn handleSchema(stream: std.net.Stream) !void { - var schema_blob = LgBlob.empty(); - var err_blob = LgBlob.empty(); - - const status = lith_introspect_schema(db, &schema_blob, &err_blob); - - if (status != .ok) { - const error_response = - \\HTTP/1.1 500 Internal Server Error - \\Content-Type: application/json - \\Connection: close - \\ - \\{"error":"introspection_failed"} - ; - _ = try stream.writeAll(error_response); - return; - } - - defer lith_blob_free(&schema_blob); - - const schema_data = schema_blob.toSlice() orelse "{}"; - - const response_header = - \\HTTP/1.1 200 OK - \\Content-Type: application/json - \\Connection: close - \\ - \\ - ; - _ = try stream.writeAll(response_header); - _ = try stream.writeAll(schema_data); -} - -fn send404(stream: std.net.Stream) !void { - const response = - \\HTTP/1.1 404 Not Found - \\Content-Type: application/json - \\Connection: close - \\ - \\{"error":"not_found"} - ; - _ = try stream.writeAll(response); -} diff --git a/lithoglyph/examples/web-project-deno.json b/lithoglyph/examples/web-project-deno.json deleted file mode 100644 index 5ddd3bd7..00000000 --- a/lithoglyph/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "^12.0.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/lithoglyph/ffi/zig/build.zig b/lithoglyph/ffi/zig/build.zig deleted file mode 100644 index af1c39bb..00000000 --- a/lithoglyph/ffi/zig/build.zig +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// build.zig - Lith FFI Build Configuration (Zig 0.15.2+) -// -// This build links the FFI layer (ffi/zig/) against the core storage engine -// (core-zig/) so that bridge.zig can delegate to the real implementation. - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // ================================================================ - // Core-zig module (the real storage engine) - // ================================================================ - - // Create a module from core-zig/src/bridge.zig so the FFI bridge - // can import it as "core_bridge" - const core_bridge_mod = b.createModule(.{ - .root_source_file = b.path("../../core-zig/src/bridge.zig"), - .target = target, - .optimize = optimize, - }); - - // ================================================================ - // FFI shared library - // ================================================================ - - const lib = b.addLibrary(.{ - .name = "lith", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - .imports = &.{ - .{ .name = "core_bridge", .module = core_bridge_mod }, - }, - }), - .linkage = .dynamic, - .version = .{ .major = 0, .minor = 6, .patch = 5 }, - }); - - // Export C symbols for FFI - lib.linkLibC(); - - b.installArtifact(lib); - - // ================================================================ - // FFI static library (for embedding) - // ================================================================ - - const static_lib = b.addLibrary(.{ - .name = "lith", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - .imports = &.{ - .{ .name = "core_bridge", .module = core_bridge_mod }, - }, - }), - .linkage = .static, - }); - - b.installArtifact(static_lib); - - // ================================================================ - // Unit tests for bridge.zig (with core-zig module available) - // ================================================================ - - const bridge_tests = b.addTest(.{ - .name = "bridge-tests", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/bridge.zig"), - .target = target, - .optimize = optimize, - .imports = &.{ - .{ .name = "core_bridge", .module = core_bridge_mod }, - }, - }), - }); - - const run_bridge_tests = b.addRunArtifact(bridge_tests); - - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_bridge_tests.step); - - // ================================================================ - // Seam tests (test integration boundaries) - // ================================================================ - - const seam_step = b.step("seam", "Run seam tests (integration boundaries)"); - seam_step.dependOn(&run_bridge_tests.step); -} diff --git a/lithoglyph/ffi/zig/src/bridge.zig b/lithoglyph/ffi/zig/src/bridge.zig deleted file mode 100644 index ca203315..00000000 --- a/lithoglyph/ffi/zig/src/bridge.zig +++ /dev/null @@ -1,771 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// bridge.zig - Idris2-facing FFI Layer for Lithoglyph Lith -// -// This module is the thin delegation layer between the Idris2 ABI definitions -// (src/abi/*.idr) and the core storage engine (core-zig/src/bridge.zig). -// -// Architecture: -// Idris2 ABI --> ffi/zig/src/bridge.zig (THIS FILE) --> core-zig/src/bridge.zig -// (type adaptation + lifecycle) (WAL, blocks, storage) -// -// The core-zig bridge provides the real storage engine with: -// - 6-phase WAL commit protocol -// - Block allocator with CRC32C checksums -// - Transaction buffering with pending writes/deletes -// - Proof verification registry (D-NORM-004) -// - Journal introspection -// -// This FFI layer adds: -// - Library-level init/cleanup lifecycle (not needed by core-zig) -// - Type adaptation between Idris2 ABI types and core-zig Lg* types -// - Collection-level operations (future, requires schema layer) -// - GQL query execution (future, requires Factor/Forth runtime) -// - Cursor-based result iteration (future, requires query engine) -// - Seam boundary tests for multi-language integration -// -// Symbol Export Strategy: -// Functions with the SAME name as core-zig exports are declared as `pub fn` -// (not `export fn`) to avoid C symbol collisions. The core-zig module's -// `export fn` declarations handle those C symbols directly. -// Functions UNIQUE to this FFI layer use `export fn` for C ABI export. -// -// CRITICAL: Pure ABI bridge - all safety logic in Idris2 - -const std = @import("std"); -const types = @import("types.zig"); -const cbor = @import("cbor.zig"); -const query_executor = @import("query_executor.zig"); - -// Import core-zig bridge (the real storage engine implementation). -// This module provides all the Lg* types and lith_* functions. -// Its `export fn` declarations will appear in the shared library's symbol table. -const core_bridge = @import("core_bridge"); - -// ============================================================ -// Re-export core-zig types for Idris2 ABI consumers -// ============================================================ - -/// Opaque database handle (delegates to core-zig LgDb) -pub const LithDb = core_bridge.LgDb; - -/// Opaque transaction handle (delegates to core-zig LgTxn) -pub const LithTxn = core_bridge.LgTxn; - -/// Core-zig types re-exported for FFI layer consumers -pub const LgBlob = core_bridge.LgBlob; -pub const LgStatus = core_bridge.LgStatus; -pub const LgResult = core_bridge.LgResult; -pub const LgRenderOpts = core_bridge.LgRenderOpts; -pub const LgTxnMode = core_bridge.LgTxnMode; -pub const LgProofVerifier = core_bridge.LgProofVerifier; - -/// Status codes (matches Idris2 LithStatus). -/// Maps to/from core-zig LgStatus internally. -pub const Status = enum(i32) { - ok = 0, - invalid_arg = 1, - not_found = 2, - permission_denied = 3, - already_exists = 4, - constraint_violation = 5, - type_mismatch = 6, - out_of_memory = 7, - io_error = 8, - corruption = 9, - conflict = 10, - internal_error = 11, -}; - -// Opaque handle types for features not yet in core-zig -pub const LithCursor = opaque {}; -pub const LithCollection = opaque {}; -pub const LithSchema = opaque {}; -pub const LithJournal = opaque {}; -pub const LithMigration = opaque {}; - -// ============================================================ -// Status Mapping: core-zig LgStatus <-> FFI Status -// ============================================================ - -/// Convert core-zig LgStatus to FFI Status code -pub fn fromLgStatus(lg_status: core_bridge.LgStatus) i32 { - return switch (lg_status) { - .ok => @intFromEnum(Status.ok), - .err_internal => @intFromEnum(Status.internal_error), - .err_not_found => @intFromEnum(Status.not_found), - .err_invalid_argument => @intFromEnum(Status.invalid_arg), - .err_out_of_memory => @intFromEnum(Status.out_of_memory), - .err_not_implemented => @intFromEnum(Status.internal_error), - .err_txn_not_active => @intFromEnum(Status.invalid_arg), - .err_txn_already_committed => @intFromEnum(Status.invalid_arg), - }; -} - -// ============================================================ -// Global State -// ============================================================ - -var initialized: bool = false; -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -var global_executor: ?query_executor.SimpleExecutor = null; - -//////////////////////////////////////////////////////////////////////////////// -// Library Lifecycle -// NOTE: core-zig does not have library-level init/cleanup; it manages state -// per database handle. This layer adds global lifecycle for the FFI consumer. -//////////////////////////////////////////////////////////////////////////////// - -/// Initialize Lith library. -/// Sets up the query executor and any global state needed by the FFI layer. -/// The core-zig storage engine is initialized per-database via lith_open. -export fn lith_init() callconv(.c) i32 { - if (initialized) return @intFromEnum(Status.ok); - - // Initialize query executor (M5 implementation) - const allocator = gpa.allocator(); - global_executor = query_executor.SimpleExecutor.init(allocator) catch { - return @intFromEnum(Status.out_of_memory); - }; - - initialized = true; - return @intFromEnum(Status.ok); -} - -/// Cleanup Lith library. -/// Tears down global state. Individual databases should be closed first. -export fn lith_cleanup() callconv(.c) void { - if (!initialized) return; - - if (global_executor) |*executor| { - executor.deinit(); - global_executor = null; - } - - _ = gpa.deinit(); - initialized = false; -} - -//////////////////////////////////////////////////////////////////////////////// -// Database Operations -// Delegates to core-zig: lith_db_open, lith_db_close -// -// These have DIFFERENT C symbol names from core-zig (lith_open vs lith_db_open), -// so they are declared as `export fn` without collision. -//////////////////////////////////////////////////////////////////////////////// - -/// Open database. -/// Delegates to core-zig/src/bridge.zig lith_db_open which handles block -/// storage initialization, superblock reading, and handle registration. -export fn lith_open( - path: [*:0]const u8, - path_len: u64, - db_out: *?*LithDb, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (path_len == 0) return @intFromEnum(Status.invalid_arg); - if (path_len > 4096) return @intFromEnum(Status.invalid_arg); - - // Delegates to core-zig/src/bridge.zig lith_db_open - var err_blob: core_bridge.LgBlob = undefined; - const status = core_bridge.lith_db_open( - @ptrCast(path), - @intCast(path_len), - null, - 0, - db_out, - &err_blob, - ); - - if (status != .ok) { - if (err_blob.ptr != null) { - core_bridge.lith_blob_free(&err_blob); - } - } - - return fromLgStatus(status); -} - -/// Close database. -/// Delegates to core-zig/src/bridge.zig lith_db_close which cleans up -/// active transactions, closes block storage, and deregisters the handle. -export fn lith_close(db: *LithDb) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - - // Clean up query executor if still active - if (global_executor) |*executor| { - executor.deinit(); - global_executor = null; - } - - // Delegates to core-zig/src/bridge.zig lith_db_close - const status = core_bridge.lith_db_close(db); - return fromLgStatus(status); -} - -/// Create new database. -/// Delegates to core-zig/src/bridge.zig lith_db_open which creates a new -/// database file if one doesn't exist (open-or-create semantics). -export fn lith_create( - path: [*:0]const u8, - path_len: u64, - block_count: u64, - db_out: *?*LithDb, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (path_len == 0) return @intFromEnum(Status.invalid_arg); - if (block_count == 0) return @intFromEnum(Status.invalid_arg); - if (block_count > 1_000_000) return @intFromEnum(Status.invalid_arg); // 4GB limit - - // Delegates to core-zig/src/bridge.zig lith_db_open - // core-zig's lith_db_open has open-or-create semantics via BlockStorage.open. - // block_count is validated above but otherwise unused: core-zig auto-grows storage. - var err_blob: core_bridge.LgBlob = undefined; - const status = core_bridge.lith_db_open( - @ptrCast(path), - @intCast(path_len), - null, - 0, - db_out, - &err_blob, - ); - - if (status != .ok) { - if (err_blob.ptr != null) { - core_bridge.lith_blob_free(&err_blob); - } - } - - return fromLgStatus(status); -} - -//////////////////////////////////////////////////////////////////////////////// -// Transaction Operations -// lith_txn_begin and lith_txn_commit have SAME C symbol names as core-zig but -// DIFFERENT signatures (FFI uses simpler i32 returns, core-zig uses LgStatus). -// To avoid linker symbol collisions, these use `export fn` with unique names -// that differ from core-zig (lith_txn_rollback vs lith_txn_abort). -// For lith_txn_begin/commit, the FFI versions shadow the core-zig versions. -//////////////////////////////////////////////////////////////////////////////// - -/// Begin transaction (FFI simplified API). -/// Delegates to core-zig/src/bridge.zig lith_txn_begin which creates a -/// TxnState with pending write/delete buffers for WAL commit protocol. -/// Uses read_write mode by default (the common case for FFI consumers). -export fn lith_ffi_txn_begin( - db: *LithDb, - txn_out: *?*LithTxn, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - - // Delegates to core-zig/src/bridge.zig lith_txn_begin - var err_blob: core_bridge.LgBlob = undefined; - const status = core_bridge.lith_txn_begin( - db, - .read_write, - txn_out, - &err_blob, - ); - - if (status != .ok) { - if (err_blob.ptr != null) { - core_bridge.lith_blob_free(&err_blob); - } - } - - return fromLgStatus(status); -} - -/// Commit transaction (FFI simplified API). -/// Delegates to core-zig/src/bridge.zig lith_txn_commit which executes the -/// 6-phase WAL commit protocol: -/// Phase 1: Write journal entries (WAL durable before data) -/// Phase 2: Sync journal to disk -/// Phase 3: Write data blocks -/// Phase 4: Process deletions -/// Phase 5: Flush superblock -/// Phase 6: Final sync -export fn lith_ffi_txn_commit(txn: *LithTxn) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - - // Delegates to core-zig/src/bridge.zig lith_txn_commit - var err_blob: core_bridge.LgBlob = undefined; - const status = core_bridge.lith_txn_commit(txn, &err_blob); - - if (status != .ok) { - if (err_blob.ptr != null) { - core_bridge.lith_blob_free(&err_blob); - } - } - - return fromLgStatus(status); -} - -/// Rollback transaction. -/// Delegates to core-zig/src/bridge.zig lith_txn_abort which discards all -/// buffered operations (nothing written to disk yet due to WAL buffering). -/// Named lith_txn_rollback (vs core-zig's lith_txn_abort) for Idris2 ABI compat. -export fn lith_txn_rollback(txn: *LithTxn) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - - // Delegates to core-zig/src/bridge.zig lith_txn_abort - const status = core_bridge.lith_txn_abort(txn); - return fromLgStatus(status); -} - -//////////////////////////////////////////////////////////////////////////////// -// Block Operations (Zig-level delegation wrappers) -// These delegate to core-zig functions with the SAME C symbol names. -// Declared as `pub fn` (not `export fn`) to avoid C symbol collisions. -// The core-zig module's `export fn` declarations provide the C ABI symbols. -// These wrappers are available for Zig-level callers within this module. -//////////////////////////////////////////////////////////////////////////////// - -/// Apply an operation (insert a new block). -/// Delegates to core-zig/src/bridge.zig lith_apply which buffers the write -/// in the transaction's pending_writes list (not durable until commit). -pub fn ffiApply( - txn: ?*LithTxn, - op_ptr: [*]const u8, - op_len: usize, -) core_bridge.LgResult { - // Delegates to core-zig/src/bridge.zig lith_apply - return core_bridge.lith_apply(txn, op_ptr, op_len); -} - -/// Update an existing block within a transaction. -/// Delegates to core-zig/src/bridge.zig lith_update_block. -pub fn ffiUpdateBlock( - txn: ?*LithTxn, - block_id: u64, - data_ptr: [*]const u8, - data_len: usize, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_update_block - return core_bridge.lith_update_block(txn, block_id, data_ptr, data_len, out_err); -} - -/// Delete a block within a transaction. -/// Delegates to core-zig/src/bridge.zig lith_delete_block. -pub fn ffiDeleteBlock( - txn: ?*LithTxn, - block_id: u64, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_delete_block - return core_bridge.lith_delete_block(txn, block_id, out_err); -} - -/// Read all blocks of a given type (full scan). -/// Delegates to core-zig/src/bridge.zig lith_read_blocks. -pub fn ffiReadBlocks( - db: ?*LithDb, - block_type: u16, - out_data: *core_bridge.LgBlob, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_read_blocks - return core_bridge.lith_read_blocks(db, block_type, out_data, out_err); -} - -//////////////////////////////////////////////////////////////////////////////// -// Introspection (Zig-level delegation wrappers) -// Same pattern: `pub fn` wrappers to avoid symbol collision with core-zig. -//////////////////////////////////////////////////////////////////////////////// - -/// Render a block as canonical text. -/// Delegates to core-zig/src/bridge.zig lith_render_block. -pub fn ffiRenderBlock( - db: ?*LithDb, - block_id: u64, - opts: core_bridge.LgRenderOpts, - out_text: *core_bridge.LgBlob, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_render_block - return core_bridge.lith_render_block(db, block_id, opts, out_text, out_err); -} - -/// Render journal entries since a sequence number. -/// Delegates to core-zig/src/bridge.zig lith_render_journal. -pub fn ffiRenderJournal( - db: ?*LithDb, - since: u64, - opts: core_bridge.LgRenderOpts, - out_text: *core_bridge.LgBlob, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_render_journal - return core_bridge.lith_render_journal(db, since, opts, out_text, out_err); -} - -/// Get database schema information. -/// Delegates to core-zig/src/bridge.zig lith_introspect_schema. -pub fn ffiIntrospectSchema( - db: ?*LithDb, - out_schema: *core_bridge.LgBlob, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_introspect_schema - return core_bridge.lith_introspect_schema(db, out_schema, out_err); -} - -/// Get constraint information. -/// Delegates to core-zig/src/bridge.zig lith_introspect_constraints. -pub fn ffiIntrospectConstraints( - db: ?*LithDb, - out_constraints: *core_bridge.LgBlob, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_introspect_constraints - return core_bridge.lith_introspect_constraints(db, out_constraints, out_err); -} - -//////////////////////////////////////////////////////////////////////////////// -// Proof Verification (Zig-level delegation wrappers, D-NORM-004) -// Same pattern: `pub fn` wrappers to avoid symbol collision with core-zig. -//////////////////////////////////////////////////////////////////////////////// - -/// Register a proof verifier for a specific proof type. -/// Delegates to core-zig/src/bridge.zig lith_proof_register_verifier. -pub fn ffiProofRegisterVerifier( - type_ptr: [*]const u8, - type_len: usize, - callback: core_bridge.LgProofVerifier, - context: ?*anyopaque, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_proof_register_verifier - return core_bridge.lith_proof_register_verifier(type_ptr, type_len, callback, context); -} - -/// Unregister a proof verifier. -/// Delegates to core-zig/src/bridge.zig lith_proof_unregister_verifier. -pub fn ffiProofUnregisterVerifier( - type_ptr: [*]const u8, - type_len: usize, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_proof_unregister_verifier - return core_bridge.lith_proof_unregister_verifier(type_ptr, type_len); -} - -/// Verify a proof using registered verifiers. -/// Delegates to core-zig/src/bridge.zig lith_proof_verify. -pub fn ffiProofVerify( - proof_ptr: [*]const u8, - proof_len: usize, - out_valid: *bool, - out_err: *core_bridge.LgBlob, -) core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_proof_verify - return core_bridge.lith_proof_verify(proof_ptr, proof_len, out_valid, out_err); -} - -/// Initialize built-in proof verifiers (fd-holds, normalization, denormalization). -/// Delegates to core-zig/src/bridge.zig lith_proof_init_builtins. -pub fn ffiProofInitBuiltins() core_bridge.LgStatus { - // Delegates to core-zig/src/bridge.zig lith_proof_init_builtins - return core_bridge.lith_proof_init_builtins(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Utility Functions (Zig-level delegation wrappers) -// Same pattern: `pub fn` wrappers to avoid symbol collision with core-zig. -//////////////////////////////////////////////////////////////////////////////// - -/// Free a blob allocated by the bridge. -/// Delegates to core-zig/src/bridge.zig lith_blob_free. -pub fn ffiBlobFree(blob: *core_bridge.LgBlob) void { - // Delegates to core-zig/src/bridge.zig lith_blob_free - core_bridge.lith_blob_free(blob); -} - -/// Get Lith version. -/// Delegates to core-zig/src/bridge.zig lith_version. -pub fn ffiVersion() u32 { - // Delegates to core-zig/src/bridge.zig lith_version - return core_bridge.lith_version(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Collection Operations -// NOT YET IMPLEMENTED: requires schema layer on top of block storage. -// Core-zig provides raw block operations; collections need schema metadata, -// document validation, and collection-level indexing. -//////////////////////////////////////////////////////////////////////////////// - -/// Create collection with schema. -/// NOT YET IMPLEMENTED: requires Idris2 schema validation layer (Proven.SafeJson) -/// and collection metadata block type support in core-zig. -export fn lith_collection_create( - db: *LithDb, - name: [*:0]const u8, - name_len: u64, - schema_json: [*:0]const u8, - schema_len: u64, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (name_len == 0) return @intFromEnum(Status.invalid_arg); - if (schema_len == 0) return @intFromEnum(Status.invalid_arg); - - // NOT YET IMPLEMENTED: requires schema layer with: - // - Collection metadata blocks (BlockType.collection_meta) - // - JSON schema validation via Idris2 Proven.SafeJson - // - Schema block allocation and linking - _ = db; - _ = name; - _ = schema_json; - return @intFromEnum(Status.internal_error); -} - -/// Drop collection. -/// NOT YET IMPLEMENTED: requires collection registry and cascade deletion -/// of all document/edge/index blocks belonging to the collection. -export fn lith_collection_drop( - db: *LithDb, - name: [*:0]const u8, - name_len: u64, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (name_len == 0) return @intFromEnum(Status.invalid_arg); - - // NOT YET IMPLEMENTED: requires collection registry and block ownership tracking - _ = db; - _ = name; - return @intFromEnum(Status.internal_error); -} - -/// Get collection schema. -/// NOT YET IMPLEMENTED: requires collection metadata block reading -/// and schema deserialization. -export fn lith_collection_schema( - db: *LithDb, - name: [*:0]const u8, - schema_out: *?*LithSchema, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - - // NOT YET IMPLEMENTED: requires schema block reading from collection metadata - _ = db; - _ = name; - _ = schema_out; - return @intFromEnum(Status.internal_error); -} - -//////////////////////////////////////////////////////////////////////////////// -// GQL Query Execution -// NOT YET IMPLEMENTED: requires Factor/Forth runtime for full query planning. -// The SimpleExecutor provides hardcoded responses for M5 testing only. -//////////////////////////////////////////////////////////////////////////////// - -/// Execute GQL query with provenance. -/// Partially implemented via SimpleExecutor (M5 hardcoded responses). -/// NOT YET IMPLEMENTED: requires Factor runtime for real query planning, -/// cursor creation from result sets, and provenance audit logging. -export fn lith_query_execute( - db: *LithDb, - query_str: [*:0]const u8, - query_len: u64, - provenance_json: [*:0]const u8, - provenance_len: u64, - cursor_out: *?*LithCursor, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (query_len == 0) return @intFromEnum(Status.invalid_arg); - if (query_len > 1_000_000) return @intFromEnum(Status.invalid_arg); // 1MB query limit - if (provenance_len == 0) return @intFromEnum(Status.invalid_arg); - - _ = db; - _ = provenance_json; // NOT YET IMPLEMENTED: provenance audit logging - - // Get the global executor (M5 hardcoded responses) - if (global_executor) |*executor| { - const query = query_str[0..query_len]; - - var result = executor.execute(query) catch { - return @intFromEnum(Status.internal_error); - }; - defer result.deinit(); - - // NOT YET IMPLEMENTED: create LithCursor from result set - _ = cursor_out; - - if (std.mem.eql(u8, result.status, "ok")) { - return @intFromEnum(Status.ok); - } else { - return @intFromEnum(Status.internal_error); - } - } - - return @intFromEnum(Status.internal_error); -} - -/// Explain GQL query (get execution plan). -/// NOT YET IMPLEMENTED: requires Factor runtime query planner with EXPLAIN -/// output generation. -export fn lith_query_explain( - db: *LithDb, - query_str: [*:0]const u8, - query_len: u64, - explain_json_out: [*]u8, - buffer_len: u64, - written_out: *u64, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (query_len == 0) return @intFromEnum(Status.invalid_arg); - if (buffer_len == 0) return @intFromEnum(Status.invalid_arg); - - // NOT YET IMPLEMENTED: requires Factor runtime query planner - _ = db; - _ = query_str; - _ = explain_json_out; - written_out.* = 0; - return @intFromEnum(Status.internal_error); -} - -//////////////////////////////////////////////////////////////////////////////// -// Cursor Operations -// NOT YET IMPLEMENTED: requires query engine result set materialization. -//////////////////////////////////////////////////////////////////////////////// - -/// Fetch next result from cursor. -/// NOT YET IMPLEMENTED: requires cursor state management and result set -/// iteration backed by block storage reads. -export fn lith_cursor_next( - cursor: *LithCursor, - document_json_out: [*]u8, - buffer_len: u64, - written_out: *u64, -) callconv(.c) i32 { - if (!initialized) return @intFromEnum(Status.internal_error); - if (buffer_len == 0) return @intFromEnum(Status.invalid_arg); - - // NOT YET IMPLEMENTED: requires cursor result set iteration - _ = cursor; - _ = document_json_out; - written_out.* = 0; - return @intFromEnum(Status.not_found); -} - -/// Close cursor. -/// NOT YET IMPLEMENTED: requires cursor state cleanup. -export fn lith_cursor_close(cursor: *LithCursor) callconv(.c) void { - if (!initialized) return; - - // NOT YET IMPLEMENTED: requires cursor state management - _ = cursor; -} - -//////////////////////////////////////////////////////////////////////////////// -// SEAM TESTING EXPORTS -// These functions verify integration boundaries between language runtimes. -// Each seam test validates that data crosses the boundary correctly. -//////////////////////////////////////////////////////////////////////////////// - -/// SEAM TEST: Verify Idris2 -> Zig boundary. -/// Tests that the core-zig bridge can be called and returns a valid status. -export fn lith_seam_test_idris_zig() callconv(.c) i32 { - // Verify core-zig bridge is callable by checking version - const version = core_bridge.lith_version(); - if (version > 0) { - return @intFromEnum(Status.ok); - } - // Version 0.0.0 would be 0, which is technically valid but unexpected - return @intFromEnum(Status.ok); -} - -/// SEAM TEST: Verify Zig -> Factor boundary. -/// NOT YET IMPLEMENTED: requires Factor runtime linkage. -export fn lith_seam_test_zig_factor() callconv(.c) i32 { - // NOT YET IMPLEMENTED: requires Factor runtime - return @intFromEnum(Status.ok); -} - -/// SEAM TEST: Verify Factor -> Forth boundary. -/// NOT YET IMPLEMENTED: requires Forth runtime linkage. -export fn lith_seam_test_factor_forth() callconv(.c) i32 { - // NOT YET IMPLEMENTED: requires Forth runtime - return @intFromEnum(Status.ok); -} - -//////////////////////////////////////////////////////////////////////////////// -// Helper Functions (ABI Bridge Only) -//////////////////////////////////////////////////////////////////////////////// - -/// Validate null-terminated C string (basic safety check). -/// NOTE: Full validation happens in Idris2 via Proven.SafeString -fn validate_c_string(ptr: [*:0]const u8, max_len: usize) bool { - var len: usize = 0; - while (ptr[len] != 0) : (len += 1) { - if (len >= max_len) return false; - } - return len > 0; -} - -//////////////////////////////////////////////////////////////////////////////// -// Tests -//////////////////////////////////////////////////////////////////////////////// - -const testing = std.testing; - -test "status codes match Idris2 ABI" { - try testing.expectEqual(@as(i32, 0), @intFromEnum(Status.ok)); - try testing.expectEqual(@as(i32, 1), @intFromEnum(Status.invalid_arg)); - try testing.expectEqual(@as(i32, 2), @intFromEnum(Status.not_found)); - try testing.expectEqual(@as(i32, 3), @intFromEnum(Status.permission_denied)); - try testing.expectEqual(@as(i32, 4), @intFromEnum(Status.already_exists)); - try testing.expectEqual(@as(i32, 5), @intFromEnum(Status.constraint_violation)); - try testing.expectEqual(@as(i32, 6), @intFromEnum(Status.type_mismatch)); - try testing.expectEqual(@as(i32, 7), @intFromEnum(Status.out_of_memory)); - try testing.expectEqual(@as(i32, 8), @intFromEnum(Status.io_error)); - try testing.expectEqual(@as(i32, 9), @intFromEnum(Status.corruption)); - try testing.expectEqual(@as(i32, 10), @intFromEnum(Status.conflict)); - try testing.expectEqual(@as(i32, 11), @intFromEnum(Status.internal_error)); -} - -test "library initialization" { - const status = lith_init(); - try testing.expectEqual(@intFromEnum(Status.ok), status); - lith_cleanup(); -} - -test "validate_c_string rejects empty strings" { - const empty_str: [*:0]const u8 = ""; - try testing.expect(!validate_c_string(empty_str, 100)); -} - -test "validate_c_string accepts valid strings" { - const valid_str: [*:0]const u8 = "SELECT * FROM users"; - try testing.expect(validate_c_string(valid_str, 1000)); -} - -test "validate_c_string rejects oversized strings" { - var buf: [200]u8 = undefined; - @memset(&buf, 'A'); - buf[199] = 0; - // SAFETY: buf is a stack-allocated [200]u8 with buf[199] = 0 (null terminator). - // The cast to [*:0]const u8 is safe because the sentinel byte is guaranteed - // present at the end. The array has byte alignment which matches u8 requirements. - const long_str: [*:0]const u8 = @ptrCast(&buf); - try testing.expect(!validate_c_string(long_str, 100)); -} - -test "core-zig version delegation" { - const version = ffiVersion(); - // core-zig returns 0.1.0 = 100 - try testing.expectEqual(@as(u32, 100), version); -} - -test "seam test idris-zig passes" { - const status = lith_seam_test_idris_zig(); - try testing.expectEqual(@intFromEnum(Status.ok), status); -} - -test "LgStatus to Status mapping" { - try testing.expectEqual(@intFromEnum(Status.ok), fromLgStatus(.ok)); - try testing.expectEqual(@intFromEnum(Status.internal_error), fromLgStatus(.err_internal)); - try testing.expectEqual(@intFromEnum(Status.not_found), fromLgStatus(.err_not_found)); - try testing.expectEqual(@intFromEnum(Status.invalid_arg), fromLgStatus(.err_invalid_argument)); - try testing.expectEqual(@intFromEnum(Status.out_of_memory), fromLgStatus(.err_out_of_memory)); -} diff --git a/lithoglyph/ffi/zig/src/cbor.zig b/lithoglyph/ffi/zig/src/cbor.zig deleted file mode 100644 index 9fd77bfa..00000000 --- a/lithoglyph/ffi/zig/src/cbor.zig +++ /dev/null @@ -1,517 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Form.Bridge - CBOR Encoding/Decoding -// -// Minimal CBOR implementation following RFC 8949. -// Supports deterministic encoding per Section 4.2. - -const std = @import("std"); -const types = @import("types.zig"); - -// ============================================================ -// CBOR Major Types -// ============================================================ - -pub const MajorType = enum(u3) { - unsigned = 0, - negative = 1, - bytes = 2, - text = 3, - array = 4, - map = 5, - tag = 6, - simple = 7, -}; - -// ============================================================ -// CBOR Encoder -// ============================================================ - -pub const Encoder = struct { - buffer: std.ArrayList(u8), - - pub fn init(allocator: std.mem.Allocator) Encoder { - return .{ - .buffer = std.ArrayList(u8).init(allocator), - }; - } - - pub fn deinit(self: *Encoder) void { - self.buffer.deinit(); - } - - pub fn finish(self: *Encoder) []const u8 { - return self.buffer.items; - } - - pub fn reset(self: *Encoder) void { - self.buffer.clearRetainingCapacity(); - } - - // Write major type with argument - fn writeTypeArg(self: *Encoder, major: MajorType, arg: u64) !void { - const base: u8 = @as(u8, @intFromEnum(major)) << 5; - - if (arg < 24) { - try self.buffer.append(base | @as(u8, @truncate(arg))); - } else if (arg <= 0xFF) { - try self.buffer.append(base | 24); - try self.buffer.append(@truncate(arg)); - } else if (arg <= 0xFFFF) { - try self.buffer.append(base | 25); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u16, @truncate(arg)))); - } else if (arg <= 0xFFFFFFFF) { - try self.buffer.append(base | 26); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u32, @truncate(arg)))); - } else { - try self.buffer.append(base | 27); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u64, arg))); - } - } - - // Encode unsigned integer - pub fn encodeUint(self: *Encoder, value: u64) !void { - try self.writeTypeArg(.unsigned, value); - } - - // Encode negative integer - pub fn encodeNint(self: *Encoder, value: i64) !void { - const n: u64 = @bitCast(-1 - value); - try self.writeTypeArg(.negative, n); - } - - // Encode integer (signed or unsigned) - pub fn encodeInt(self: *Encoder, value: i64) !void { - if (value >= 0) { - try self.encodeUint(@bitCast(value)); - } else { - try self.encodeNint(value); - } - } - - // Encode byte string - pub fn encodeBytes(self: *Encoder, data: []const u8) !void { - try self.writeTypeArg(.bytes, data.len); - try self.buffer.appendSlice(data); - } - - // Encode text string - pub fn encodeText(self: *Encoder, text: []const u8) !void { - try self.writeTypeArg(.text, text.len); - try self.buffer.appendSlice(text); - } - - // Begin array (definite length) - pub fn beginArray(self: *Encoder, len: usize) !void { - try self.writeTypeArg(.array, len); - } - - // Begin map (definite length) - pub fn beginMap(self: *Encoder, len: usize) !void { - try self.writeTypeArg(.map, len); - } - - // Encode tag - pub fn encodeTag(self: *Encoder, tag: u64) !void { - try self.writeTypeArg(.tag, tag); - } - - // Encode Lith-specific tag - pub fn encodeLithTag(self: *Encoder, tag: types.CborTag) !void { - try self.encodeTag(@intFromEnum(tag)); - } - - // Encode null - pub fn encodeNull(self: *Encoder) !void { - try self.buffer.append(0xF6); - } - - // Encode boolean - pub fn encodeBool(self: *Encoder, value: bool) !void { - try self.buffer.append(if (value) 0xF5 else 0xF4); - } - - // Encode float (smallest representation per RFC 8949 §4.2) - pub fn encodeFloat(self: *Encoder, value: f64) !void { - // Check if it fits in half precision - const half: f16 = @floatCast(value); - if (@as(f64, @floatCast(half)) == value) { - try self.buffer.append(0xF9); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u16, @bitCast(half)))); - return; - } - - // Check if it fits in single precision - const single: f32 = @floatCast(value); - if (@as(f64, @floatCast(single)) == value) { - try self.buffer.append(0xFA); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u32, @bitCast(single)))); - return; - } - - // Use double precision - try self.buffer.append(0xFB); - try self.buffer.appendSlice(&std.mem.toBytes(std.mem.nativeToBig(u64, @bitCast(value)))); - } - - // Encode a simple document (map of string -> any) - pub fn encodeDocument(self: *Encoder, fields: anytype) !void { - const info = @typeInfo(@TypeOf(fields)); - const struct_info = info.@"struct"; - - try self.beginMap(struct_info.fields.len); - - inline for (struct_info.fields) |field| { - try self.encodeText(field.name); - const value = @field(fields, field.name); - try self.encodeValue(value); - } - } - - // Encode any value (comptime type dispatch) - pub fn encodeValue(self: *Encoder, value: anytype) !void { - const T = @TypeOf(value); - - if (T == bool) { - try self.encodeBool(value); - } else if (@typeInfo(T) == .int) { - try self.encodeInt(@intCast(value)); - } else if (@typeInfo(T) == .float) { - try self.encodeFloat(@floatCast(value)); - } else if (T == []const u8) { - try self.encodeText(value); - } else if (@typeInfo(T) == .pointer) { - if (@typeInfo(T).pointer.size == .Slice) { - if (@typeInfo(T).pointer.child == u8) { - try self.encodeText(value); - } else { - try self.beginArray(value.len); - for (value) |item| { - try self.encodeValue(item); - } - } - } - } else if (@typeInfo(T) == .optional) { - if (value) |v| { - try self.encodeValue(v); - } else { - try self.encodeNull(); - } - } else if (@typeInfo(T) == .@"struct") { - try self.encodeDocument(value); - } else { - @compileError("unsupported type for CBOR encoding: " ++ @typeName(T)); - } - } -}; - -// ============================================================ -// CBOR Decoder -// ============================================================ - -pub const DecodeError = error{ - UnexpectedEof, - InvalidType, - InvalidValue, - OutOfMemory, -}; - -pub const Decoder = struct { - data: []const u8, - pos: usize, - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator, data: []const u8) Decoder { - return .{ - .data = data, - .pos = 0, - .allocator = allocator, - }; - } - - fn remaining(self: *Decoder) []const u8 { - return self.data[self.pos..]; - } - - fn readByte(self: *Decoder) !u8 { - if (self.pos >= self.data.len) return error.UnexpectedEof; - const b = self.data[self.pos]; - self.pos += 1; - return b; - } - - fn readBytes(self: *Decoder, n: usize) ![]const u8 { - if (self.pos + n > self.data.len) return error.UnexpectedEof; - const slice = self.data[self.pos .. self.pos + n]; - self.pos += n; - return slice; - } - - fn readArg(self: *Decoder, additional: u5) !u64 { - if (additional < 24) return additional; - switch (additional) { - 24 => return try self.readByte(), - 25 => { - const bytes = try self.readBytes(2); - return std.mem.bigToNative(u16, std.mem.bytesToValue(u16, bytes[0..2])); - }, - 26 => { - const bytes = try self.readBytes(4); - return std.mem.bigToNative(u32, std.mem.bytesToValue(u32, bytes[0..4])); - }, - 27 => { - const bytes = try self.readBytes(8); - return std.mem.bigToNative(u64, std.mem.bytesToValue(u64, bytes[0..8])); - }, - else => return error.InvalidValue, - } - } - - pub fn readTypeArg(self: *Decoder) !struct { major: MajorType, arg: u64 } { - const b = try self.readByte(); - const major: MajorType = @enumFromInt(@as(u3, @truncate(b >> 5))); - const additional: u5 = @truncate(b); - const arg = try self.readArg(additional); - return .{ .major = major, .arg = arg }; - } - - pub fn decodeUint(self: *Decoder) !u64 { - const ta = try self.readTypeArg(); - if (ta.major != .unsigned) return error.InvalidType; - return ta.arg; - } - - pub fn decodeInt(self: *Decoder) !i64 { - const ta = try self.readTypeArg(); - switch (ta.major) { - .unsigned => return @bitCast(ta.arg), - .negative => return -1 - @as(i64, @bitCast(ta.arg)), - else => return error.InvalidType, - } - } - - pub fn decodeText(self: *Decoder) ![]const u8 { - const ta = try self.readTypeArg(); - if (ta.major != .text) return error.InvalidType; - return try self.readBytes(@intCast(ta.arg)); - } - - pub fn decodeBytes(self: *Decoder) ![]const u8 { - const ta = try self.readTypeArg(); - if (ta.major != .bytes) return error.InvalidType; - return try self.readBytes(@intCast(ta.arg)); - } - - pub fn decodeArrayLen(self: *Decoder) !usize { - const ta = try self.readTypeArg(); - if (ta.major != .array) return error.InvalidType; - return @intCast(ta.arg); - } - - pub fn decodeMapLen(self: *Decoder) !usize { - const ta = try self.readTypeArg(); - if (ta.major != .map) return error.InvalidType; - return @intCast(ta.arg); - } - - pub fn decodeTag(self: *Decoder) !u64 { - const ta = try self.readTypeArg(); - if (ta.major != .tag) return error.InvalidType; - return ta.arg; - } - - pub fn decodeBool(self: *Decoder) !bool { - const b = try self.readByte(); - switch (b) { - 0xF4 => return false, - 0xF5 => return true, - else => return error.InvalidType, - } - } - - pub fn isNull(self: *Decoder) !bool { - if (self.pos >= self.data.len) return error.UnexpectedEof; - if (self.data[self.pos] == 0xF6) { - self.pos += 1; - return true; - } - return false; - } - - pub fn skip(self: *Decoder) !void { - const ta = try self.readTypeArg(); - switch (ta.major) { - .unsigned, .negative => {}, - .bytes, .text => { - _ = try self.readBytes(@intCast(ta.arg)); - }, - .array => { - var i: usize = 0; - while (i < ta.arg) : (i += 1) { - try self.skip(); - } - }, - .map => { - var i: usize = 0; - while (i < ta.arg) : (i += 1) { - try self.skip(); // key - try self.skip(); // value - } - }, - .tag => { - try self.skip(); - }, - .simple => { - // Handle floats - const additional: u5 = @truncate(self.data[self.pos - 1]); - switch (additional) { - 25 => _ = try self.readBytes(2), - 26 => _ = try self.readBytes(4), - 27 => _ = try self.readBytes(8), - else => {}, - } - }, - } - } -}; - -// ============================================================ -// Helper Functions -// ============================================================ - -// Encode a provenance payload -pub fn encodeProvenance( - allocator: std.mem.Allocator, - actor_id: []const u8, - actor_type: []const u8, - rationale: []const u8, - timestamp: []const u8, -) ![]u8 { - var encoder = Encoder.init(allocator); - errdefer encoder.deinit(); - - try encoder.encodeLithTag(.provenance); - try encoder.beginMap(3); - - // actor - try encoder.encodeText("actor"); - try encoder.encodeLithTag(.actor); - try encoder.beginMap(2); - try encoder.encodeText("id"); - try encoder.encodeText(actor_id); - try encoder.encodeText("type"); - try encoder.encodeText(actor_type); - - // rationale - try encoder.encodeText("rationale"); - try encoder.encodeText(rationale); - - // timestamp - try encoder.encodeText("timestamp"); - try encoder.encodeTag(0); // datetime tag - try encoder.encodeText(timestamp); - - const result = try allocator.dupe(u8, encoder.finish()); - encoder.deinit(); - return result; -} - -// Encode an error blob -pub fn encodeError( - allocator: std.mem.Allocator, - code: i32, - message: []const u8, -) ![]u8 { - var encoder = Encoder.init(allocator); - errdefer encoder.deinit(); - - try encoder.beginMap(2); - try encoder.encodeText("code"); - try encoder.encodeInt(code); - try encoder.encodeText("message"); - try encoder.encodeText(message); - - const result = try allocator.dupe(u8, encoder.finish()); - encoder.deinit(); - return result; -} - -// ============================================================ -// Tests -// ============================================================ - -test "encode unsigned integers" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.encodeUint(0); - try encoder.encodeUint(23); - try encoder.encodeUint(24); - try encoder.encodeUint(255); - try encoder.encodeUint(256); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0x00), result[0]); // 0 - try std.testing.expectEqual(@as(u8, 0x17), result[1]); // 23 - try std.testing.expectEqual(@as(u8, 0x18), result[2]); // 24 prefix - try std.testing.expectEqual(@as(u8, 0x18), result[3]); // 24 value - try std.testing.expectEqual(@as(u8, 0x18), result[4]); // 255 prefix - try std.testing.expectEqual(@as(u8, 0xFF), result[5]); // 255 value -} - -test "encode text string" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.encodeText("hello"); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0x65), result[0]); // text(5) - try std.testing.expectEqualStrings("hello", result[1..6]); -} - -test "encode simple map" { - var encoder = Encoder.init(std.testing.allocator); - defer encoder.deinit(); - - try encoder.beginMap(2); - try encoder.encodeText("name"); - try encoder.encodeText("Lith"); - try encoder.encodeText("version"); - try encoder.encodeUint(1); - - const result = encoder.finish(); - - try std.testing.expectEqual(@as(u8, 0xA2), result[0]); // map(2) -} - -test "decode unsigned integer" { - const data = [_]u8{ 0x18, 0x64 }; // 100 - var decoder = Decoder.init(std.testing.allocator, &data); - - const value = try decoder.decodeUint(); - try std.testing.expectEqual(@as(u64, 100), value); -} - -test "decode text string" { - const data = [_]u8{ 0x65, 'h', 'e', 'l', 'l', 'o' }; - var decoder = Decoder.init(std.testing.allocator, &data); - - const text = try decoder.decodeText(); - try std.testing.expectEqualStrings("hello", text); -} - -test "encode provenance" { - const result = try encodeProvenance( - std.testing.allocator, - "user_123", - "human", - "Adding test data", - "2026-01-11T12:00:00Z", - ); - defer std.testing.allocator.free(result); - - // Verify it starts with provenance tag - try std.testing.expectEqual(@as(u8, 0xD9), result[0]); // tag (2-byte) -} diff --git a/lithoglyph/ffi/zig/src/main.zig b/lithoglyph/ffi/zig/src/main.zig deleted file mode 100644 index 6b233bc7..00000000 --- a/lithoglyph/ffi/zig/src/main.zig +++ /dev/null @@ -1,274 +0,0 @@ -// {{PROJECT}} FFI Implementation -// -// This module implements the C-compatible FFI declared in src/abi/Foreign.idr -// All types and layouts must match the Idris2 ABI definitions. -// -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -// Version information (keep in sync with project) -const VERSION = "0.1.0"; -const BUILD_INFO = "{{PROJECT}} built with Zig " ++ @import("builtin").zig_version_string; - -/// Thread-local error storage -threadlocal var last_error: ?[]const u8 = null; - -/// Set the last error message -fn setError(msg: []const u8) void { - last_error = msg; -} - -/// Clear the last error -fn clearError() void { - last_error = null; -} - -//============================================================================== -// Core Types (must match src/abi/Types.idr) -//============================================================================== - -/// Result codes (must match Idris2 Result type) -pub const Result = enum(c_int) { - ok = 0, - @"error" = 1, - invalid_param = 2, - out_of_memory = 3, - null_pointer = 4, -}; - -/// Library handle (opaque to prevent direct access) -pub const Handle = opaque { - // Internal state hidden from C - allocator: std.mem.Allocator, - initialized: bool, - // Add your fields here -}; - -//============================================================================== -// Library Lifecycle -//============================================================================== - -/// Initialize the library -/// Returns a handle, or null on failure -export fn {{project}}_init() ?*Handle { - const allocator = std.heap.c_allocator; - - const handle = allocator.create(Handle) catch { - setError("Failed to allocate handle"); - return null; - }; - - // Initialize handle - handle.* = .{ - .allocator = allocator, - .initialized = true, - }; - - clearError(); - return handle; -} - -/// Free the library handle -export fn {{project}}_free(handle: ?*Handle) void { - const h = handle orelse return; - const allocator = h.allocator; - - // Clean up resources - h.initialized = false; - - allocator.destroy(h); - clearError(); -} - -//============================================================================== -// Core Operations -//============================================================================== - -/// Process data (example operation) -export fn {{project}}_process(handle: ?*Handle, input: u32) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Example processing logic - _ = input; - - clearError(); - return .ok; -} - -//============================================================================== -// String Operations -//============================================================================== - -/// Get a string result (example) -/// Caller must free the returned string -export fn {{project}}_get_string(handle: ?*Handle) ?[*:0]const u8 { - const h = handle orelse { - setError("Null handle"); - return null; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return null; - } - - // Example: allocate and return a string - const result = h.allocator.dupeZ(u8, "Example result") catch { - setError("Failed to allocate string"); - return null; - }; - - clearError(); - return result.ptr; -} - -/// Free a string allocated by the library -export fn {{project}}_free_string(str: ?[*:0]const u8) void { - const s = str orelse return; - const allocator = std.heap.c_allocator; - - const slice = std.mem.span(s); - allocator.free(slice); -} - -//============================================================================== -// Array/Buffer Operations -//============================================================================== - -/// Process an array of data -export fn {{project}}_process_array( - handle: ?*Handle, - buffer: ?[*]const u8, - len: u32, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const buf = buffer orelse { - setError("Null buffer"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Access the buffer - const data = buf[0..len]; - _ = data; - - // Process data here - - clearError(); - return .ok; -} - -//============================================================================== -// Error Handling -//============================================================================== - -/// Get the last error message -/// Returns null if no error -export fn {{project}}_last_error() ?[*:0]const u8 { - const err = last_error orelse return null; - - // Return C string (static storage, no need to free) - const allocator = std.heap.c_allocator; - const c_str = allocator.dupeZ(u8, err) catch return null; - return c_str.ptr; -} - -//============================================================================== -// Version Information -//============================================================================== - -/// Get the library version -export fn {{project}}_version() [*:0]const u8 { - return VERSION.ptr; -} - -/// Get build information -export fn {{project}}_build_info() [*:0]const u8 { - return BUILD_INFO.ptr; -} - -//============================================================================== -// Callback Support -//============================================================================== - -/// Callback function type (C ABI) -pub const Callback = *const fn (u64, u32) callconv(.C) u32; - -/// Register a callback -export fn {{project}}_register_callback( - handle: ?*Handle, - callback: ?Callback, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const cb = callback orelse { - setError("Null callback"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Store callback for later use - _ = cb; - - clearError(); - return .ok; -} - -//============================================================================== -// Utility Functions -//============================================================================== - -/// Check if handle is initialized -export fn {{project}}_is_initialized(handle: ?*Handle) u32 { - const h = handle orelse return 0; - return if (h.initialized) 1 else 0; -} - -//============================================================================== -// Tests -//============================================================================== - -test "lifecycle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try std.testing.expect({{project}}_is_initialized(handle) == 1); -} - -test "error handling" { - const result = {{project}}_process(null, 0); - try std.testing.expectEqual(Result.null_pointer, result); - - const err = {{project}}_last_error(); - try std.testing.expect(err != null); -} - -test "version" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - try std.testing.expectEqualStrings(VERSION, ver_str); -} diff --git a/lithoglyph/ffi/zig/src/query_executor.zig b/lithoglyph/ffi/zig/src/query_executor.zig deleted file mode 100644 index f334009b..00000000 --- a/lithoglyph/ffi/zig/src/query_executor.zig +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// query_executor.zig - Simple GQL Query Executor in Zig -// -// This is a minimal implementation for M5. Later it will call Factor runtime. - -const std = @import("std"); -const json = std.json; - -pub const QueryResult = struct { - status: []const u8, - data: ?std.json.Parsed(std.json.Value) = null, - error_message: ?[]const u8 = null, - allocator: std.mem.Allocator, - - pub fn deinit(self: *QueryResult) void { - if (self.data) |*d| { - d.deinit(); - } - } -}; - -pub const SimpleExecutor = struct { - allocator: std.mem.Allocator, - // In-memory storage for M5 (will be replaced by Forth in M6) - collections: std.StringHashMap(std.ArrayList(std.json.Value)), - - pub fn init(allocator: std.mem.Allocator) !SimpleExecutor { - return SimpleExecutor{ - .allocator = allocator, - .collections = std.StringHashMap(std.ArrayList(std.json.Value)).init(allocator), - }; - } - - pub fn deinit(self: *SimpleExecutor) void { - var it = self.collections.iterator(); - while (it.next()) |entry| { - entry.value_ptr.deinit(self.allocator); - } - self.collections.deinit(); - } - - /// Execute a simple GQL query (hardcoded for M5) - pub fn execute(self: *SimpleExecutor, query: []const u8) !QueryResult { - // Simple query parsing - just check for keywords - if (std.mem.indexOf(u8, query, "SELECT") != null) { - return try self.executeSelect(query); - } else if (std.mem.indexOf(u8, query, "INSERT") != null) { - return try self.executeInsert(query); - } else if (std.mem.indexOf(u8, query, "CREATE") != null) { - return try self.executeCreate(query); - } else { - return QueryResult{ - .status = "error", - .error_message = "Unknown query type", - .allocator = self.allocator, - }; - } - } - - fn executeSelect(self: *SimpleExecutor, query: []const u8) !QueryResult { - _ = query; - - // Hardcoded response for M5 testing - const response = - \\{ - \\ "status": "ok", - \\ "collection": "evidence", - \\ "count": 1, - \\ "rows": [ - \\ { - \\ "id": "1", - \\ "title": "Test Evidence", - \\ "score": 95 - \\ } - \\ ] - \\} - ; - - const parsed = try std.json.parseFromSlice( - std.json.Value, - self.allocator, - response, - .{}, - ); - - return QueryResult{ - .status = "ok", - .data = parsed, - .allocator = self.allocator, - }; - } - - fn executeInsert(self: *SimpleExecutor, query: []const u8) !QueryResult { - _ = query; - - // Hardcoded response for M5 testing - const response = - \\{ - \\ "status": "ok", - \\ "document_id": "abc123", - \\ "collection": "evidence" - \\} - ; - - const parsed = try std.json.parseFromSlice( - std.json.Value, - self.allocator, - response, - .{}, - ); - - return QueryResult{ - .status = "ok", - .data = parsed, - .allocator = self.allocator, - }; - } - - fn executeCreate(self: *SimpleExecutor, query: []const u8) !QueryResult { - _ = query; - - // Hardcoded response for M5 testing - const response = - \\{ - \\ "status": "ok", - \\ "collection": "evidence", - \\ "schema_version": 1 - \\} - ; - - const parsed = try std.json.parseFromSlice( - std.json.Value, - self.allocator, - response, - .{}, - ); - - return QueryResult{ - .status = "ok", - .data = parsed, - .allocator = self.allocator, - }; - } -}; - -test "simple executor - select" { - const testing = std.testing; - const allocator = testing.allocator; - - var executor = try SimpleExecutor.init(allocator); - defer executor.deinit(); - - var result = try executor.execute("SELECT * FROM evidence"); - defer result.deinit(); - - try testing.expectEqualStrings("ok", result.status); - try testing.expect(result.data != null); -} - -test "simple executor - insert" { - const testing = std.testing; - const allocator = testing.allocator; - - var executor = try SimpleExecutor.init(allocator); - defer executor.deinit(); - - var result = try executor.execute("INSERT INTO evidence VALUES (...)"); - defer result.deinit(); - - try testing.expectEqualStrings("ok", result.status); -} diff --git a/lithoglyph/ffi/zig/src/types.zig b/lithoglyph/ffi/zig/src/types.zig deleted file mode 100644 index 3780a5b3..00000000 --- a/lithoglyph/ffi/zig/src/types.zig +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Form.Bridge - Type Definitions - -const std = @import("std"); - -// ============================================================ -// Blob Encoding -// ============================================================ - -pub const BlobEncoding = enum(u8) { - cbor = 0, - cbor_compressed = 1, - reserved = 255, -}; - -// ============================================================ -// Blob Structure -// ============================================================ - -pub const LithBlob = extern struct { - data: ?[*]const u8, - len: usize, - encoding: BlobEncoding, - _padding: [7]u8 = [_]u8{0} ** 7, - - pub fn empty() LithBlob { - return .{ - .data = null, - .len = 0, - .encoding = .cbor, - }; - } - - pub fn fromSlice(slice: []const u8) LithBlob { - return .{ - .data = slice.ptr, - .len = slice.len, - .encoding = .cbor, - }; - } - - pub fn toSlice(self: LithBlob) ?[]const u8 { - if (self.data) |ptr| { - return ptr[0..self.len]; - } - return null; - } -}; - -// ============================================================ -// Status Codes -// ============================================================ - -pub const LithStatus = enum(i32) { - ok = 0, - - // Database errors (1xxx) - err_db_not_found = 1001, - err_db_already_open = 1002, - err_db_corrupted = 1003, - err_db_version_mismatch = 1004, - - // Transaction errors (2xxx) - err_txn_not_active = 2001, - err_txn_already_committed = 2002, - err_txn_already_aborted = 2003, - err_txn_conflict = 2004, - - // Document errors (3xxx) - err_doc_not_found = 3001, - err_doc_already_exists = 3002, - err_doc_validation_failed = 3003, - - // Collection errors (4xxx) - err_collection_not_found = 4001, - err_collection_already_exists = 4002, - - // Schema errors (5xxx) - err_schema_violation = 5001, - err_constraint_violation = 5002, - - // Internal errors (9xxx) - err_internal = 9001, - err_out_of_memory = 9002, - err_invalid_argument = 9003, - err_not_implemented = 9004, - - pub fn isOk(self: LithStatus) bool { - return self == .ok; - } - - pub fn isError(self: LithStatus) bool { - return @intFromEnum(self) > 0; - } -}; - -// ============================================================ -// Transaction Mode -// ============================================================ - -pub const LithTxnMode = enum(u8) { - read_only = 0, - read_write = 1, -}; - -// ============================================================ -// Operation Types -// ============================================================ - -pub const LithOpType = enum(u16) { - // Document operations - doc_insert = 0x0001, - doc_update = 0x0002, - doc_delete = 0x0003, - doc_replace = 0x0004, - - // Edge operations - edge_insert = 0x0010, - edge_delete = 0x0011, - edge_update = 0x0012, - - // Collection operations - collection_create = 0x0020, - collection_drop = 0x0021, - - // Schema operations - schema_create = 0x0030, - schema_alter = 0x0031, - - // Constraint operations - constraint_add = 0x0040, - constraint_drop = 0x0041, - - // Index operations - index_create = 0x0050, - index_drop = 0x0051, - - // Query operations - query_select = 0x0100, - query_aggregate = 0x0101, - query_explain = 0x0102, -}; - -// ============================================================ -// Block Types -// ============================================================ - -pub const LithBlockType = enum(u16) { - free = 0x0000, - superblock = 0x0001, - collection_meta = 0x0010, - document = 0x0011, - document_overflow = 0x0012, - edge_meta = 0x0020, - edge = 0x0021, - index_root = 0x0030, - index_internal = 0x0031, - index_leaf = 0x0032, - journal_segment = 0x0040, - schema = 0x0050, - constraint = 0x0051, - migration = 0x0060, -}; - -// ============================================================ -// CBOR Tags (Lith-specific) -// ============================================================ - -pub const CborTag = enum(u64) { - datetime = 0, - uri = 32, - self_described = 55799, - block_reference = 39001, - document_id = 39002, - collection_name = 39003, - provenance = 39004, - actor = 39005, - prompt_score = 39006, - functional_dependency = 39007, - proof = 39008, -}; - -// ============================================================ -// Render Options -// ============================================================ - -pub const LithRenderOpts = extern struct { - include_provenance: bool = true, - include_timestamps: bool = true, - pretty_print: bool = false, - max_depth: u32 = 10, - _padding: [3]u8 = [_]u8{0} ** 3, -}; - -// ============================================================ -// Result Structure -// ============================================================ - -pub const LithResult = extern struct { - result_blob: LithBlob, - provenance_blob: LithBlob, - status: LithStatus, - _padding: [4]u8 = [_]u8{0} ** 4, - err_blob: LithBlob, - - pub fn ok(result: LithBlob) LithResult { - return .{ - .result_blob = result, - .provenance_blob = LithBlob.empty(), - .status = .ok, - .err_blob = LithBlob.empty(), - }; - } - - pub fn okWithProvenance(result: LithBlob, provenance: LithBlob) LithResult { - return .{ - .result_blob = result, - .provenance_blob = provenance, - .status = .ok, - .err_blob = LithBlob.empty(), - }; - } - - pub fn err(status: LithStatus, err_blob: LithBlob) LithResult { - return .{ - .result_blob = LithBlob.empty(), - .provenance_blob = LithBlob.empty(), - .status = status, - .err_blob = err_blob, - }; - } -}; - -// ============================================================ -// Tests -// ============================================================ - -test "LithBlob empty" { - const blob = LithBlob.empty(); - try std.testing.expectEqual(@as(?[*]const u8, null), blob.data); - try std.testing.expectEqual(@as(usize, 0), blob.len); -} - -test "LithBlob fromSlice" { - const data = "test data"; - const blob = LithBlob.fromSlice(data); - try std.testing.expectEqual(@as(usize, 9), blob.len); - - if (blob.toSlice()) |slice| { - try std.testing.expectEqualStrings("test data", slice); - } else { - try std.testing.expect(false); - } -} - -test "LithStatus" { - try std.testing.expect(LithStatus.ok.isOk()); - try std.testing.expect(!LithStatus.ok.isError()); - try std.testing.expect(!LithStatus.err_doc_not_found.isOk()); - try std.testing.expect(LithStatus.err_doc_not_found.isError()); -} diff --git a/lithoglyph/ffi/zig/test/integration_test.zig b/lithoglyph/ffi/zig/test/integration_test.zig deleted file mode 100644 index 03419949..00000000 --- a/lithoglyph/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,182 +0,0 @@ -// {{PROJECT}} Integration Tests -// SPDX-License-Identifier: MPL-2.0 -// -// These tests verify that the Zig FFI correctly implements the Idris2 ABI - -const std = @import("std"); -const testing = std.testing; - -// Import FFI functions -extern fn {{project}}_init() ?*opaque {}; -extern fn {{project}}_free(?*opaque {}) void; -extern fn {{project}}_process(?*opaque {}, u32) c_int; -extern fn {{project}}_get_string(?*opaque {}) ?[*:0]const u8; -extern fn {{project}}_free_string(?[*:0]const u8) void; -extern fn {{project}}_last_error() ?[*:0]const u8; -extern fn {{project}}_version() [*:0]const u8; -extern fn {{project}}_is_initialized(?*opaque {}) u32; - -//============================================================================== -// Lifecycle Tests -//============================================================================== - -test "create and destroy handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try testing.expect(handle != null); -} - -test "handle is initialized" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const initialized = {{project}}_is_initialized(handle); - try testing.expectEqual(@as(u32, 1), initialized); -} - -test "null handle is not initialized" { - const initialized = {{project}}_is_initialized(null); - try testing.expectEqual(@as(u32, 0), initialized); -} - -//============================================================================== -// Operation Tests -//============================================================================== - -test "process with valid handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const result = {{project}}_process(handle, 42); - try testing.expectEqual(@as(c_int, 0), result); // 0 = ok -} - -test "process with null handle returns error" { - const result = {{project}}_process(null, 42); - try testing.expectEqual(@as(c_int, 4), result); // 4 = null_pointer -} - -//============================================================================== -// String Tests -//============================================================================== - -test "get string result" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const str = {{project}}_get_string(handle); - defer if (str) |s| {{project}}_free_string(s); - - try testing.expect(str != null); -} - -test "get string with null handle" { - const str = {{project}}_get_string(null); - try testing.expect(str == null); -} - -//============================================================================== -// Error Handling Tests -//============================================================================== - -test "last error after null handle operation" { - _ = {{project}}_process(null, 0); - - const err = {{project}}_last_error(); - try testing.expect(err != null); - - if (err) |e| { - const err_str = std.mem.span(e); - try testing.expect(err_str.len > 0); - } -} - -test "no error after successful operation" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - _ = {{project}}_process(handle, 0); - - // Error should be cleared after successful operation - // (This depends on implementation) -} - -//============================================================================== -// Version Tests -//============================================================================== - -test "version string is not empty" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - try testing.expect(ver_str.len > 0); -} - -test "version string is semantic version format" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - // Should be in format X.Y.Z - try testing.expect(std.mem.count(u8, ver_str, ".") >= 1); -} - -//============================================================================== -// Memory Safety Tests -//============================================================================== - -test "multiple handles are independent" { - const h1 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h1); - - const h2 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h2); - - try testing.expect(h1 != h2); - - // Operations on h1 should not affect h2 - _ = {{project}}_process(h1, 1); - _ = {{project}}_process(h2, 2); -} - -test "double free is safe" { - const handle = {{project}}_init() orelse return error.InitFailed; - - {{project}}_free(handle); - {{project}}_free(handle); // Should not crash -} - -test "free null is safe" { - {{project}}_free(null); // Should not crash -} - -//============================================================================== -// Thread Safety Tests (if applicable) -//============================================================================== - -test "concurrent operations" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const ThreadContext = struct { - h: *opaque {}, - id: u32, - }; - - const thread_fn = struct { - fn run(ctx: ThreadContext) void { - _ = {{project}}_process(ctx.h, ctx.id); - } - }.run; - - var threads: [4]std.Thread = undefined; - for (&threads, 0..) |*thread, i| { - thread.* = try std.Thread.spawn(.{}, thread_fn, .{ - ThreadContext{ .h = handle, .id = @intCast(i) }, - }); - } - - for (threads) |thread| { - thread.join(); - } -} diff --git a/lithoglyph/generated/abi/bridge.h b/lithoglyph/generated/abi/bridge.h deleted file mode 100644 index 6b2c6168..00000000 --- a/lithoglyph/generated/abi/bridge.h +++ /dev/null @@ -1,385 +0,0 @@ -/* SPDX-License-Identifier: MPL-2.0 */ -/* Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) */ -/* - * Lithoglyph Bridge C Header - * Generated from Idris2 ABI definitions (src/Lith/) - * - * DO NOT EDIT — regenerate from ABI - * - * This header defines the stable C ABI for all runtimes (Factor, Forth, - * Erlang/BEAM) to interact with the Lithoglyph storage engine via Zig. - */ - -#ifndef LITHOGLYPH_BRIDGE_H -#define LITHOGLYPH_BRIDGE_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* ============================================================ - * Status Codes (Lith.LithBridge.LithStatus) - * - * Unified superset of core-zig LgStatus (0-7) and - * ffi/zig Status (0-11). Values 0-7 are implemented; - * values 8-11 are reserved for future use. - * ============================================================ */ -typedef enum { - LITH_OK = 0, - LITH_ERR_INTERNAL = 1, - LITH_ERR_NOT_FOUND = 2, - LITH_ERR_INVALID_ARGUMENT = 3, - LITH_ERR_OUT_OF_MEMORY = 4, - LITH_ERR_NOT_IMPLEMENTED = 5, - LITH_ERR_TXN_NOT_ACTIVE = 6, - LITH_ERR_TXN_ALREADY_COMMITTED = 7, - /* Reserved (ffi/zig extended codes) */ - LITH_ERR_IO_ERROR = 8, - LITH_ERR_CORRUPTION = 9, - LITH_ERR_CONFLICT = 10, - LITH_ERR_ALREADY_EXISTS = 11, -} LithStatus; - -/* ============================================================ - * Opaque Handles - * ============================================================ */ -typedef struct LithDb LithDb; -typedef struct LithTxn LithTxn; - -/* ============================================================ - * Blob Types (Lith.LithBridge + core-zig) - * ============================================================ */ - -/** Owned byte buffer passed across the FFI boundary */ -typedef struct { - const uint8_t* ptr; - size_t len; -} LgBlob; - -/** Result type for operations returning data + provenance */ -typedef struct { - LgBlob data; - LgBlob provenance; - int status; /* LithStatus */ - LgBlob error_blob; -} LgResult; - -/** Transaction mode */ -typedef enum { - LG_TXN_READ_ONLY = 0, - LG_TXN_READ_WRITE = 1, -} LgTxnMode; - -/** Render options for introspection functions */ -typedef struct { - int format; /* 0 = JSON */ - bool include_metadata; -} LgRenderOpts; - -/** Proof verifier callback type */ -typedef LithStatus (*LgProofVerifier)( - const uint8_t* proof_ptr, - size_t proof_len, - void* context -); - -/* ============================================================ - * Constants (Lith.LithBridge + LithLayout) - * ============================================================ */ - -/** Block size in bytes (4 KiB) */ -#define LG_BLOCK_SIZE 4096 -/** Block header size in bytes */ -#define LG_BLOCK_HEADER_SIZE 64 -/** Block payload size in bytes */ -#define LG_BLOCK_PAYLOAD_SIZE 4032 -/** Block type: document */ -#define LG_BLOCK_TYPE_DOCUMENT 0x0011 - -/* ============================================================ - * Implemented Functions (core-zig/src/bridge.zig) - * - * These are the working bridge functions. Signatures match - * the Idris2 ABI declarations in LithForeign.idr. - * ============================================================ */ - -/* --- Database Lifecycle --- */ - -/** - * Open a Lith database. - * - * @param path_ptr Path to database file - * @param path_len Length of path - * @param opts_ptr CBOR-encoded options (may be NULL) - * @param opts_len Length of options (0 if opts_ptr is NULL) - * @param out_db Output: database handle - * @param out_err Output: error blob (empty on success) - * @return LithStatus - */ -LithStatus lith_db_open( - const uint8_t* path_ptr, size_t path_len, - const uint8_t* opts_ptr, size_t opts_len, - LithDb** out_db, LgBlob* out_err -); - -/** - * Close a Lith database and release resources. - * - * @param db Database handle (may be NULL — returns INVALID_ARGUMENT) - * @return LithStatus - */ -LithStatus lith_db_close(LithDb* db); - -/* --- Transaction Management --- */ - -/** - * Begin a new transaction. - * - * @param db Database handle - * @param mode Transaction mode (read-only or read-write) - * @param out_txn Output: transaction handle - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_txn_begin( - LithDb* db, LgTxnMode mode, - LithTxn** out_txn, LgBlob* out_err -); - -/** - * Commit a transaction (6-phase WAL: journal → sync → blocks → deletes → superblock → sync). - * - * @param txn Transaction handle - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_txn_commit(LithTxn* txn, LgBlob* out_err); - -/** - * Abort a transaction, discarding all buffered operations. - * - * @param txn Transaction handle - * @return LithStatus - */ -LithStatus lith_txn_abort(LithTxn* txn); - -/* --- Operations (buffered until commit) --- */ - -/** - * Apply an insert operation within a transaction. - * Data is buffered and not written to disk until commit. - * - * @param txn Transaction handle - * @param op_ptr Operation data (JSON document) - * @param op_len Length of operation data - * @return LgResult with block_id in data blob on success - */ -LgResult lith_apply(LithTxn* txn, const uint8_t* op_ptr, size_t op_len); - -/** - * Update an existing block within a transaction. - * - * @param txn Transaction handle - * @param block_id Block ID to update - * @param data_ptr New data - * @param data_len Length of new data - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_update_block( - LithTxn* txn, uint64_t block_id, - const uint8_t* data_ptr, size_t data_len, - LgBlob* out_err -); - -/** - * Delete a block within a transaction. - * - * @param txn Transaction handle - * @param block_id Block ID to delete - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_delete_block(LithTxn* txn, uint64_t block_id, LgBlob* out_err); - -/* --- Query --- */ - -/** - * Read all blocks of a given type (full scan). - * Returns a JSON array of objects with block_id, size, and data fields. - * - * @param db Database handle - * @param block_type Block type filter (e.g. LG_BLOCK_TYPE_DOCUMENT) - * @param out_data Output: JSON array blob - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_read_blocks( - LithDb* db, uint16_t block_type, - LgBlob* out_data, LgBlob* out_err -); - -/* --- Introspection --- */ - -/** - * Render a block as canonical text (JSON). - * - * @param db Database handle - * @param block_id Block ID to render - * @param opts Render options - * @param out_text Output: text blob - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_render_block( - LithDb* db, uint64_t block_id, - LgRenderOpts opts, - LgBlob* out_text, LgBlob* out_err -); - -/** - * Render journal entries since a sequence number. - * - * @param db Database handle - * @param since Starting sequence number - * @param opts Render options - * @param out_text Output: text blob - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_render_journal( - LithDb* db, uint64_t since, - LgRenderOpts opts, - LgBlob* out_text, LgBlob* out_err -); - -/** - * Get database schema information as JSON. - * - * @param db Database handle - * @param out_schema Output: schema blob - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_introspect_schema( - LithDb* db, LgBlob* out_schema, LgBlob* out_err -); - -/** - * Get constraint information as JSON. - * - * @param db Database handle - * @param out_constraints Output: constraints blob - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_introspect_constraints( - LithDb* db, LgBlob* out_constraints, LgBlob* out_err -); - -/* --- Proof Verification --- */ - -/** - * Register a proof verifier for a specific proof type. - * - * @param type_ptr Proof type identifier (e.g. "fd-holds", "normalization") - * @param type_len Length of type identifier - * @param callback Verification callback function - * @param context Optional context passed to callback (may be NULL) - * @return LithStatus - */ -LithStatus lith_proof_register_verifier( - const uint8_t* type_ptr, size_t type_len, - LgProofVerifier callback, void* context -); - -/** - * Unregister a proof verifier. - * - * @param type_ptr Proof type identifier - * @param type_len Length of type identifier - * @return LithStatus (NOT_FOUND if not registered) - */ -LithStatus lith_proof_unregister_verifier( - const uint8_t* type_ptr, size_t type_len -); - -/** - * Verify a proof using registered verifiers. - * Expects JSON: {"type":"proof_type","data":"base64_data"} - * - * @param proof_ptr JSON-encoded proof blob - * @param proof_len Length of proof - * @param out_valid Output: true if proof is valid - * @param out_err Output: error blob - * @return LithStatus - */ -LithStatus lith_proof_verify( - const uint8_t* proof_ptr, size_t proof_len, - bool* out_valid, LgBlob* out_err -); - -/** - * Initialize built-in proof verifiers (fd-holds, normalization, denormalization). - * - * @return LithStatus - */ -LithStatus lith_proof_init_builtins(void); - -/* --- Utilities --- */ - -/** - * Free a blob allocated by the bridge. - * - * @param blob Blob to free (ptr set to NULL after free) - */ -void lith_blob_free(LgBlob* blob); - -/** - * Get Lith version as encoded integer. - * Format: major * 10000 + minor * 100 + patch - * Example: 0.1.0 = 100 - * - * @return Version number - */ -uint32_t lith_version(void); - -/* ============================================================ - * Planned Functions (not yet implemented in core-zig) - * - * These are declared in LithForeign.idr (liblith) but - * not yet available in the core bridge. Uncomment as - * implementations land. - * ============================================================ */ - -/* LithStatus lith_init(void); */ -/* void lith_cleanup(void); */ -/* LithStatus lith_create(const char* path, size_t path_len, uint64_t block_count, LithDb** out_db); */ -/* LithStatus lith_collection_create(LithDb* db, const char* name, size_t name_len, const char* schema_json, size_t schema_len); */ -/* LithStatus lith_collection_drop(LithDb* db, const char* name, size_t name_len); */ -/* LithStatus lith_collection_schema(LithDb* db, const char* name, void** schema_out); */ -/* LithStatus lith_query_execute(LithDb* db, const char* query, size_t query_len, const char* provenance, size_t prov_len, void** cursor_out); */ -/* LithStatus lith_query_explain(LithDb* db, const char* query, size_t query_len, void* buf, size_t buf_len, size_t* written); */ -/* LithStatus lith_cursor_next(void* cursor, void* buf, size_t buf_len, size_t* written); */ -/* void lith_cursor_close(void* cursor); */ -/* LithStatus lith_journal_get(LithDb* db, void** journal_out); */ -/* LithStatus lith_journal_read(void* journal, uint64_t start_seq, uint64_t count, void* buf, size_t buf_len, size_t* written); */ -/* LithStatus lith_journal_replay(LithDb* db, uint64_t from_seq); */ -/* LithStatus lith_normalize_discover(LithDb* db, const char* collection, void* buf, size_t buf_len, size_t* written); */ -/* LithStatus lith_normalize_analyze(LithDb* db, const char* collection, void* nf_out); */ -/* LithStatus lith_migrate_start(LithDb* db, const char* collection, uint8_t target_nf, void* proof, size_t proof_len, void** migration_out); */ -/* LithStatus lith_migrate_commit(void* migration, uint8_t phase); */ -/* LithStatus lith_serialize_cbor(const char* json, size_t json_len, void* buf, size_t buf_len, size_t* written); */ -/* LithStatus lith_deserialize_cbor(void* cbor, size_t cbor_len, void* buf, size_t buf_len, size_t* written); */ -/* LithStatus lith_verify_checksums(LithDb* db, void* corrupted_out, size_t buf_len, size_t* count_out); */ -/* LithStatus lith_repair(LithDb* db, void* report_buf, size_t buf_len, size_t* written); */ - -#ifdef __cplusplus -} -#endif - -#endif /* LITHOGLYPH_BRIDGE_H */ diff --git a/lithoglyph/geo/.clusterfuzzlite/Containerfile b/lithoglyph/geo/.clusterfuzzlite/Containerfile deleted file mode 100644 index ace36012..00000000 --- a/lithoglyph/geo/.clusterfuzzlite/Containerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM gcr.io/oss-fuzz-base/base-builder-rust -RUN apt-get update && apt-get install -y make autoconf automake libtool -COPY . $SRC/project -WORKDIR $SRC/project -COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/lithoglyph/geo/.clusterfuzzlite/build.sh b/lithoglyph/geo/.clusterfuzzlite/build.sh deleted file mode 100755 index a60fba6d..00000000 --- a/lithoglyph/geo/.clusterfuzzlite/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -eu - -cd $SRC/project -cargo +nightly fuzz build --release -cp fuzz/target/*/release/fuzz_* $OUT/ diff --git a/lithoglyph/geo/.clusterfuzzlite/project.yaml b/lithoglyph/geo/.clusterfuzzlite/project.yaml deleted file mode 100644 index 4bb8843a..00000000 --- a/lithoglyph/geo/.clusterfuzzlite/project.yaml +++ /dev/null @@ -1,4 +0,0 @@ -language: rust -sanitizers: - - address - - undefined diff --git a/lithoglyph/geo/.editorconfig b/lithoglyph/geo/.editorconfig deleted file mode 100644 index fc6650ce..00000000 --- a/lithoglyph/geo/.editorconfig +++ /dev/null @@ -1,68 +0,0 @@ -# RSR-template-repo - Editor Configuration -# https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 2 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -trim_trailing_whitespace = false - -[*.adoc] -trim_trailing_whitespace = false - -[*.rs] -indent_size = 4 - -[*.ex] -indent_size = 2 - -[*.exs] -indent_size = 2 - -[*.zig] -indent_size = 4 - -[*.ada] -indent_size = 3 - -[*.adb] -indent_size = 3 - -[*.ads] -indent_size = 3 - -[*.hs] -indent_size = 2 - -[*.res] -indent_size = 2 - -[*.resi] -indent_size = 2 - -[*.ncl] -indent_size = 2 - -[*.rkt] -indent_size = 2 - -[*.scm] -indent_size = 2 - -[*.nix] -indent_size = 2 - -[Justfile] -indent_style = space -indent_size = 4 - -[justfile] -indent_style = space -indent_size = 4 diff --git a/lithoglyph/geo/.gitattributes b/lithoglyph/geo/.gitattributes deleted file mode 100644 index e860a85c..00000000 --- a/lithoglyph/geo/.gitattributes +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitattributes - -* text=auto eol=lf - -# Source -*.rs text eol=lf diff=rust -*.ex text eol=lf diff=elixir -*.exs text eol=lf diff=elixir -*.jl text eol=lf -*.res text eol=lf -*.resi text eol=lf -*.ada text eol=lf diff=ada -*.adb text eol=lf diff=ada -*.ads text eol=lf diff=ada -*.hs text eol=lf -*.chpl text eol=lf -*.scm text eol=lf -*.ncl text eol=lf -*.nix text eol=lf - -# Docs -*.md text eol=lf diff=markdown -*.adoc text eol=lf -*.txt text eol=lf - -# Data -*.json text eol=lf -*.yaml text eol=lf -*.yml text eol=lf -*.toml text eol=lf - -# Config -.gitignore text eol=lf -.gitattributes text eol=lf -justfile text eol=lf -Makefile text eol=lf -Containerfile text eol=lf - -# Scripts -*.sh text eol=lf - -# Binary -*.png binary -*.jpg binary -*.gif binary -*.pdf binary -*.woff2 binary -*.zip binary -*.gz binary - -# Lock files -Cargo.lock text eol=lf -diff -flake.lock text eol=lf -diff diff --git a/lithoglyph/geo/.gitignore b/lithoglyph/geo/.gitignore deleted file mode 100644 index c01969fb..00000000 --- a/lithoglyph/geo/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Cargo -/target/ -Cargo.lock - -# IDE -.idea/ -.vscode/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Local config (contains secrets) -config.local.toml - -# Test artifacts -*.profraw -coverage/ -target/ -node_modules/ -_build/ -deps/ -.elixir_ls/ -.cache/ -*.log -*.tmp -build/ -dist/ diff --git a/lithoglyph/geo/.machine_readable/6a2/ECOSYSTEM.a2ml b/lithoglyph/geo/.machine_readable/6a2/ECOSYSTEM.a2ml deleted file mode 100644 index b41d7a75..00000000 --- a/lithoglyph/geo/.machine_readable/6a2/ECOSYSTEM.a2ml +++ /dev/null @@ -1,60 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith-Geo Ecosystem Definition -; Media Type: application/vnd.ecosystem+scm - -(ecosystem - (version "1.0.0") - (name "lithoglyph-geo") - (type "projection-service") - (purpose "Spatial indexing and geospatial queries over Lith documents") - - (position-in-ecosystem - (layer "query-projection") - (role "Provides R-tree spatial indexing as a materialized projection over Lith") - (data-flow "Lith → lithoglyph-geo (read-only projection)") - (integration-point "HTTP API consuming Lith collections")) - - (related-projects - (project - (name "lith") - (relationship "upstream-dependency") - (description "Source of truth for all document data") - (integration "HTTP API fetch from collections")) - - (project - (name "lithoglyph-studio") - (relationship "sibling-service") - (description "GUI for Lith - may consume geo API for map visualization") - (integration "Could call /geo/* endpoints for location-based UI")) - - (project - (name "lithoglyph-analytics") - (relationship "sibling-service") - (description "OLAP analytics layer - may use geo data for spatial aggregations") - (integration "Could share spatial projections")) - - (project - (name "bofig") - (relationship "potential-consumer") - (description "Evidence graph for journalism with location-tagged evidence") - (integration "Could use geo queries for location-based evidence retrieval")) - - (project - (name "anamnesis") - (relationship "potential-consumer") - (description "Conversation knowledge extraction with location context") - (integration "Could enrich conversations with spatial relationships"))) - - (what-this-is - ("R-tree spatial index over Lith documents") - ("Materialized projection - Lith remains source of truth") - ("HTTP API for bounding box, radius, and nearest-neighbor queries") - ("Haversine distance calculations for geographic accuracy") - ("Automatic reindexing from Lith on demand")) - - (what-this-is-not - ("Not a spatial database - just an index/projection") - ("Not a data store - reads from Lith, stores nothing persistent") - ("Not a replacement for PostGIS or similar - lightweight projection only") - ("Not geospatial analysis - just spatial queries") - ("Not responsible for data integrity - Lith handles that"))) diff --git a/lithoglyph/geo/.machine_readable/6a2/META.a2ml b/lithoglyph/geo/.machine_readable/6a2/META.a2ml deleted file mode 100644 index ad1cd946..00000000 --- a/lithoglyph/geo/.machine_readable/6a2/META.a2ml +++ /dev/null @@ -1,98 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith-Geo Meta Information -; Media Type: application/meta+scheme - -(meta - (architecture-decisions - (adr - (id "adr-001") - (title "Use R-tree for spatial indexing") - (status "accepted") - (date "2025-01-16") - (context "Need efficient spatial queries (bbox, radius, nearest) over geographic points") - (decision "Use rstar crate which provides a pure-Rust R-tree implementation with bulk loading") - (consequences - ("O(log n) query performance for spatial operations") - ("Memory-resident index requires rebuild on restart") - ("Well-tested, production-ready implementation"))) - - (adr - (id "adr-002") - (title "Projection architecture - Lith as source of truth") - (status "accepted") - (date "2025-01-16") - (context "Lith philosophy prioritizes auditability over performance; spatial queries need fast lookups") - (decision "lithoglyph-geo is a materialized projection that reads from Lith but does not persist spatial data independently") - (consequences - ("Spatial index must be rebuilt from Lith on restart") - ("No risk of data divergence - Lith is always authoritative") - ("Aligns with Lith's reversibility principle"))) - - (adr - (id "adr-003") - (title "Haversine distance for geographic calculations") - (status "accepted") - (date "2025-01-16") - (context "Need accurate distance calculations on Earth's surface for radius queries") - (decision "Use Haversine formula via geo crate for distance calculations") - (consequences - ("Accurate for short to medium distances") - ("Slightly less accurate than Vincenty for very long distances") - ("Well-understood, standard approach"))) - - (adr - (id "adr-004") - (title "HTTP API with axum") - (status "accepted") - (date "2025-01-16") - (context "Need to expose spatial queries as a service that other Lith ecosystem tools can consume") - (decision "Use axum framework for async HTTP API") - (consequences - ("Modern, ergonomic Rust web framework") - ("Good integration with tokio runtime") - ("Type-safe request extraction"))) - - (adr - (id "adr-005") - (title "Support multiple location formats") - (status "accepted") - (date "2025-01-16") - (context "Lith documents may store locations in various formats") - (decision "Support object format {lat, lon}, array format [lon, lat], and GeoJSON Point") - (consequences - ("Flexibility for document authors") - ("GeoJSON order is [lon, lat] which differs from common intuition") - ("Must document supported formats clearly")))) - - (development-practices - (code-style - (formatter "rustfmt") - (linter "clippy") - (edition "2021")) - (security - (dependencies "Audit with cargo-audit") - (input-validation "Validate all query parameters") - (rate-limiting "TODO: Add rate limiting for production")) - (testing - (unit-tests "In module with #[cfg(test)]") - (integration-tests "tests/ directory") - (coverage-target 80)) - (versioning "Semantic versioning") - (documentation - (readme "README.adoc") - (api-docs "OpenAPI 3.0 (planned)") - (inline "/// doc comments")) - (branching - (main "main - stable releases") - (develop "dev - integration branch") - (features "feat/* - feature branches"))) - - (design-rationale - (why-rust - "Performance-critical spatial operations benefit from Rust's zero-cost abstractions. The rstar R-tree implementation is pure Rust with excellent performance characteristics.") - (why-separate-repo - "Lith's philosophy explicitly deprioritizes performance (Auditability > Performance). Spatial queries require optimized data structures. Separation maintains Lith's principles while enabling fast spatial lookups.") - (why-in-memory-index - "Spatial data is derived from Lith, so persistence would duplicate data. In-memory index with rebuild-on-demand keeps the system simple and prevents data divergence.") - (why-http-api - "HTTP provides language-agnostic integration. lithoglyph-studio (ReScript), lithoglyph-analytics (Julia), and other tools can all consume the same spatial API."))) diff --git a/lithoglyph/geo/.machine_readable/6a2/STATE.a2ml b/lithoglyph/geo/.machine_readable/6a2/STATE.a2ml deleted file mode 100644 index c62877a1..00000000 --- a/lithoglyph/geo/.machine_readable/6a2/STATE.a2ml +++ /dev/null @@ -1,132 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith-Geo Project State -; Reference: hyperpolymath/git-hud/STATE.scm - -(state - (metadata - (version "0.1.0") - (schema-version "1.0") - (created "2025-01-16") - (updated "2025-01-16") - (project "lithoglyph-geo") - (repo "https://github.com/hyperpolymath/lithoglyph-geo")) - - (project-context - (name "Lith-Geo") - (tagline "Spatial projection layer for Lith") - (tech-stack - (primary "Rust") - (frameworks ("axum" "tokio" "rstar")) - (dependencies ("geo" "geojson" "reqwest" "serde")))) - - (current-position - (phase "initial-implementation") - (overall-completion 15) - (components - (component - (name "spatial-index") - (completion 80) - (status "core-implemented") - (notes "R-tree with bbox, radius, nearest queries")) - (component - (name "lithoglyph-client") - (completion 70) - (status "core-implemented") - (notes "HTTP client with location extraction")) - (component - (name "http-api") - (completion 60) - (status "handlers-implemented") - (notes "All endpoints defined, needs testing")) - (component - (name "config") - (completion 90) - (status "complete") - (notes "TOML config with defaults")) - (component - (name "testing") - (completion 20) - (status "unit-tests-only") - (notes "Basic unit tests, needs integration tests")) - (component - (name "documentation") - (completion 50) - (status "readme-complete") - (notes "README done, needs API docs"))) - (working-features - ("R-tree spatial indexing") - ("Bounding box queries") - ("Radius queries with Haversine") - ("K-nearest neighbor queries") - ("Lith document fetching") - ("Location extraction from multiple formats"))) - - (route-to-mvp - (milestone - (name "compile-and-run") - (status "pending") - (items - ("Verify Cargo.toml dependencies resolve") - ("Fix any compilation errors") - ("Run with default config"))) - (milestone - (name "lithoglyph-integration") - (status "pending") - (items - ("Test against real Lith instance") - ("Verify document fetching") - ("Validate location extraction"))) - (milestone - (name "api-testing") - (status "pending") - (items - ("Test /geo/health endpoint") - ("Test /geo/reindex endpoint") - ("Test all query endpoints") - ("Load test with sample data"))) - (milestone - (name "documentation") - (status "pending") - (items - ("OpenAPI spec generation") - ("Example curl commands") - ("Docker deployment guide")))) - - (blockers-and-issues - (critical ()) - (high - (issue - (id "GEO-001") - (description "Need Lith instance for integration testing") - (mitigation "Can use mock server initially"))) - (medium - (issue - (id "GEO-002") - (description "Auto-rebuild scheduler not implemented") - (mitigation "Manual POST /geo/reindex works"))) - (low ())) - - (critical-next-actions - (immediate - ("Compile and verify builds") - ("Create sample config file") - ("Test against Lith")) - (this-week - ("Integration tests") - ("Docker deployment") - ("CI/CD setup")) - (this-month - ("Performance benchmarks") - ("OpenAPI documentation") - ("Coordinate with lithoglyph-studio for map viz"))) - - (session-history - (snapshot - (date "2025-01-16") - (accomplishments - ("Created initial Rust project structure") - ("Implemented R-tree spatial index") - ("Implemented Lith HTTP client") - ("Created axum HTTP API") - ("Added configuration system") - ("Created README and ECOSYSTEM files"))))) diff --git a/lithoglyph/geo/ABI-FFI-README.md b/lithoglyph/geo/ABI-FFI-README.md deleted file mode 100644 index e6a32bbf..00000000 --- a/lithoglyph/geo/ABI-FFI-README.md +++ /dev/null @@ -1,385 +0,0 @@ -{{~ Aditionally delete this line and fill out the template below ~}} - -# {{PROJECT}} ABI/FFI Documentation - -## Overview - -This library follows the **Hyperpolymath RSR Standard** for ABI and FFI design: - -- **ABI (Application Binary Interface)** defined in **Idris2** with formal proofs -- **FFI (Foreign Function Interface)** implemented in **Zig** for C compatibility -- **Generated C headers** bridge Idris2 ABI to Zig FFI -- **Any language** can call through standard C ABI - -## Architecture - -``` -┌─────────────────────────────────────────────┐ -│ ABI Definitions (Idris2) │ -│ src/abi/ │ -│ - Types.idr (Type definitions) │ -│ - Layout.idr (Memory layout proofs) │ -│ - Foreign.idr (FFI declarations) │ -└─────────────────┬───────────────────────────┘ - │ - │ generates (at compile time) - ▼ -┌─────────────────────────────────────────────┐ -│ C Headers (auto-generated) │ -│ generated/abi/{{project}}.h │ -└─────────────────┬───────────────────────────┘ - │ - │ imported by - ▼ -┌─────────────────────────────────────────────┐ -│ FFI Implementation (Zig) │ -│ ffi/zig/src/main.zig │ -│ - Implements C-compatible functions │ -│ - Zero-cost abstractions │ -│ - Memory-safe by default │ -└─────────────────┬───────────────────────────┘ - │ - │ compiled to lib{{project}}.so/.a - ▼ -┌─────────────────────────────────────────────┐ -│ Any Language via C ABI │ -│ - Rust, ReScript, Julia, Python, etc. │ -└─────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -{{project}}/ -├── src/ -│ ├── abi/ # ABI definitions (Idris2) -│ │ ├── Types.idr # Core type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI function declarations -│ └── lib/ # Core library (any language) -│ -├── ffi/ -│ └── zig/ # FFI implementation (Zig) -│ ├── build.zig # Build configuration -│ ├── build.zig.zon # Dependencies -│ ├── src/ -│ │ └── main.zig # C-compatible FFI implementation -│ ├── test/ -│ │ └── integration_test.zig -│ └── include/ -│ └── {{project}}.h # C header (optional, can be generated) -│ -├── generated/ # Auto-generated files -│ └── abi/ -│ └── {{project}}.h # Generated from Idris2 ABI -│ -└── bindings/ # Language-specific wrappers (optional) - ├── rust/ - ├── rescript/ - └── julia/ -``` - -## Why Idris2 for ABI? - -### 1. **Formal Verification** - -Idris2's dependent types allow proving properties about the ABI at compile-time: - -```idris --- Prove struct size is correct -public export -exampleStructSize : HasSize ExampleStruct 16 - --- Prove field alignment is correct -public export -fieldAligned : Divides 8 (offsetOf ExampleStruct.field) - --- Prove ABI is platform-compatible -public export -abiCompatible : Compatible (ABI 1) (ABI 2) -``` - -### 2. **Type Safety** - -Encode invariants that C/Zig cannot express: - -```idris --- Non-null pointer guaranteed at type level -data Handle : Type where - MkHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> Handle - --- Array with length proof -data Buffer : (n : Nat) -> Type where - MkBuffer : Vect n Byte -> Buffer n -``` - -### 3. **Platform Abstraction** - -Platform-specific types with compile-time selection: - -```idris -CInt : Platform -> Type -CInt Linux = Bits32 -CInt Windows = Bits32 - -CSize : Platform -> Type -CSize Linux = Bits64 -CSize Windows = Bits64 -``` - -### 4. **Safe Evolution** - -Prove that new ABI versions are backward-compatible: - -```idris --- Compiler enforces compatibility -abiUpgrade : ABI 1 -> ABI 2 -abiUpgrade old = MkABI2 { - -- Must preserve all v1 fields - v1_compat = old, - -- Can add new fields - new_features = defaults -} -``` - -## Why Zig for FFI? - -### 1. **C ABI Compatibility** - -Zig exports C-compatible functions naturally: - -```zig -export fn library_function(param: i32) i32 { - return param * 2; -} -``` - -### 2. **Memory Safety** - -Compile-time safety without runtime overhead: - -```zig -// Null check enforced at compile time -const handle = init() orelse return error.InitFailed; -defer free(handle); -``` - -### 3. **Cross-Compilation** - -Built-in cross-compilation to any platform: - -```bash -zig build -Dtarget=x86_64-linux -zig build -Dtarget=aarch64-macos -zig build -Dtarget=x86_64-windows -``` - -### 4. **Zero Dependencies** - -No runtime, no libc required (unless explicitly needed): - -```zig -// Minimal binary size -pub const lib = @import("std"); -// Only includes what you use -``` - -## Building - -### Build FFI Library - -```bash -cd ffi/zig -zig build # Build debug -zig build -Doptimize=ReleaseFast # Build optimized -zig build test # Run tests -``` - -### Generate C Header from Idris2 ABI - -```bash -cd src/abi -idris2 --cg c-header Types.idr -o ../../generated/abi/{{project}}.h -``` - -### Cross-Compile - -```bash -cd ffi/zig - -# Linux x86_64 -zig build -Dtarget=x86_64-linux - -# macOS ARM64 -zig build -Dtarget=aarch64-macos - -# Windows x86_64 -zig build -Dtarget=x86_64-windows -``` - -## Usage - -### From C - -```c -#include "{{project}}.h" - -int main() { - void* handle = {{project}}_init(); - if (!handle) return 1; - - int result = {{project}}_process(handle, 42); - if (result != 0) { - const char* err = {{project}}_last_error(); - fprintf(stderr, "Error: %s\n", err); - } - - {{project}}_free(handle); - return 0; -} -``` - -Compile with: -```bash -gcc -o example example.c -l{{project}} -L./zig-out/lib -``` - -### From Idris2 - -```idris -import {{PROJECT}}.ABI.Foreign - -main : IO () -main = do - Just handle <- init - | Nothing => putStrLn "Failed to initialize" - - Right result <- process handle 42 - | Left err => putStrLn $ "Error: " ++ errorDescription err - - free handle - putStrLn "Success" -``` - -### From Rust - -```rust -#[link(name = "{{project}}")] -extern "C" { - fn {{project}}_init() -> *mut std::ffi::c_void; - fn {{project}}_free(handle: *mut std::ffi::c_void); - fn {{project}}_process(handle: *mut std::ffi::c_void, input: u32) -> i32; -} - -fn main() { - unsafe { - let handle = {{project}}_init(); - assert!(!handle.is_null()); - - let result = {{project}}_process(handle, 42); - assert_eq!(result, 0); - - {{project}}_free(handle); - } -} -``` - -### From Julia - -```julia -const lib{{project}} = "lib{{project}}" - -function init() - handle = ccall((:{{project}}_init, lib{{project}}), Ptr{Cvoid}, ()) - handle == C_NULL && error("Failed to initialize") - handle -end - -function process(handle, input) - result = ccall((:{{project}}_process, lib{{project}}), Cint, (Ptr{Cvoid}, UInt32), handle, input) - result -end - -function cleanup(handle) - ccall((:{{project}}_free, lib{{project}}), Cvoid, (Ptr{Cvoid},), handle) -end - -# Usage -handle = init() -try - result = process(handle, 42) - println("Result: $result") -finally - cleanup(handle) -end -``` - -## Testing - -### Unit Tests (Zig) - -```bash -cd ffi/zig -zig build test -``` - -### Integration Tests - -```bash -cd ffi/zig -zig build test-integration -``` - -### ABI Verification (Idris2) - -```idris --- Compile-time verification -%runElab verifyABI - --- Runtime checks -main : IO () -main = do - verifyLayoutsCorrect - verifyAlignmentsCorrect - putStrLn "ABI verification passed" -``` - -## Contributing - -When modifying the ABI/FFI: - -1. **Update ABI first** (`src/abi/*.idr`) - - Modify type definitions - - Update proofs - - Ensure backward compatibility - -2. **Generate C header** - ```bash - idris2 --cg c-header src/abi/Types.idr -o generated/abi/{{project}}.h - ``` - -3. **Update FFI implementation** (`ffi/zig/src/main.zig`) - - Implement new functions - - Match ABI types exactly - -4. **Add tests** - - Unit tests in Zig - - Integration tests - - ABI verification tests - -5. **Update documentation** - - Function signatures - - Usage examples - - Migration guide (if breaking changes) - -## License - -PMPL-1.0-or-later - -## See Also - -- [Idris2 Documentation](https://idris2.readthedocs.io) -- [Zig Documentation](https://ziglang.org/documentation/master/) -- [Rhodium Standard Repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [FFI Migration Guide](../ffi-migration-guide.md) -- [ABI Migration Guide](../abi-migration-guide.md) diff --git a/lithoglyph/geo/Cargo.toml b/lithoglyph/geo/Cargo.toml deleted file mode 100644 index e062996c..00000000 --- a/lithoglyph/geo/Cargo.toml +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -[package] -name = "lithoglyph-geo" -version = "0.1.0" -edition = "2021" -authors = ["Jonathan D.A. Jewell "] -description = "Geospatial extension for Lith - spatial indexing without compromising auditability" -license = "MPL-2.0" -repository = "https://github.com/hyperpolymath/lithoglyph-geo" -keywords = ["lithoglyph", "geospatial", "spatial-index", "r-tree", "gis"] -categories = ["database", "science::geo"] - -[dependencies] -# Async runtime -tokio = { version = "1.35", features = ["full"] } - -# HTTP server -axum = "0.7" -tower = "0.4" -tower-http = { version = "0.5", features = ["cors", "trace"] } - -# Spatial indexing -rstar = "0.12" -geo = "0.28" -geojson = "0.24" - -# HTTP client (for Lith API) -reqwest = { version = "0.11", features = ["json"] } - -# Serialization -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" - -# Date/time -chrono = { version = "0.4", features = ["serde"] } - -# Configuration -toml = "0.8" -config = "0.14" - -# Logging -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } - -# Error handling -thiserror = "1.0" -anyhow = "1.0" - -# CLI -clap = { version = "4.4", features = ["derive"] } - -[dev-dependencies] -# Testing -tokio-test = "0.4" -assert_cmd = "2.0" -predicates = "3.0" - -[profile.release] -lto = true -codegen-units = 1 -panic = "abort" -strip = true diff --git a/lithoglyph/geo/LICENSE b/lithoglyph/geo/LICENSE deleted file mode 100644 index ec540b34..00000000 --- a/lithoglyph/geo/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: MPL-2.0 -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/lithoglyph/geo/MAINTAINERS.adoc b/lithoglyph/geo/MAINTAINERS.adoc deleted file mode 100644 index 48d97817..00000000 --- a/lithoglyph/geo/MAINTAINERS.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Maintainers -:toc: preamble - -This document lists the maintainers of this project and their responsibilities. - -== Current Maintainers - -[cols="2,3,2",options="header"] -|=== -| Name | Role | Contact - -| Jonathan D.A. Jewell -| Lead Maintainer -| https://github.com/hyperpolymath[@hyperpolymath] -|=== - -== Responsibilities - -Maintainers are responsible for: - -* Reviewing and merging pull requests -* Triaging issues and feature requests -* Ensuring code quality and security standards -* Managing releases and versioning -* Upholding the project's code of conduct - -== Becoming a Maintainer - -Contributors who demonstrate: - -* Consistent, high-quality contributions -* Understanding of the project's goals and standards -* Constructive participation in discussions -* Commitment to the project's long-term health - -May be invited to become maintainers at the discretion of existing maintainers. - -== Decision Making - -* Routine decisions (bug fixes, minor improvements) can be made by any maintainer -* Significant changes require discussion and consensus among maintainers -* Breaking changes or major features should be discussed in issues before implementation - -== Contact - -For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/lithoglyph/geo/MOVED.md b/lithoglyph/geo/MOVED.md deleted file mode 100644 index 4fea7859..00000000 --- a/lithoglyph/geo/MOVED.md +++ /dev/null @@ -1,12 +0,0 @@ -# This Repository Has Moved - -**This component is now part of the [Lithoglyph monorepo](https://github.com/hyperpolymath/lith).** - -## New Location - -- **Monorepo:** https://github.com/hyperpolymath/lith -- **This Component:** https://github.com/hyperpolymath/lith/tree/main/geogeoextensions/geogeo - -All Lithoglyph development now happens in the monorepo. - -**See you at [github.com/hyperpolymath/lith](https://github.com/hyperpolymath/lith)!** 🚀 diff --git a/lithoglyph/geo/README.adoc b/lithoglyph/geo/README.adoc deleted file mode 100644 index 4e0a42c3..00000000 --- a/lithoglyph/geo/README.adoc +++ /dev/null @@ -1,299 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2026 hyperpolymath -= Lith-Geo -:toc: macro -:toc-title: Contents -:toclevels: 3 -:icons: font - -image:https://img.shields.io/badge/Stage-Development-orange.svg[Stage] -image:https://img.shields.io/badge/License-MPL--2.0-blue.svg[License: PMPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] - -[.lead] -*Geospatial extension for Lith - spatial indexing without compromising auditability.* - -Location-aware evidence tracking for investigative journalism. - -toc::[] - -== Overview - -Lith-Geo adds geospatial capabilities to Lith while preserving the core guarantees: - -* **Auditability preserved** - Spatial operations are projections, source data remains in Lith -* **Reversibility maintained** - Spatial index is derived, can be rebuilt from journal -* **Provenance tracked** - "Where did this location data come from?" - -=== Use Cases - -* **Investigative journalism**: Map evidence locations, track movement patterns -* **Compliance**: Geographic jurisdiction tracking, data sovereignty -* **Cultural archives**: Historical location mapping, site preservation - -== Architecture - -[source,text] ----- -┌─────────────────────────────────────────────────────────────┐ -│ Lith-Geo API │ -│ REST endpoints for spatial queries │ -├─────────────────────────────────────────────────────────────┤ -│ Spatial Index (R-tree) │ -│ Materialized projection of Lith data │ -│ Rebuilt on demand from Lith journal │ -├─────────────────────────────────────────────────────────────┤ -│ Lith HTTP API (reads) │ -│ All writes go through Lith (provenance preserved) │ -└─────────────────────────────────────────────────────────────┘ ----- - -=== Key Principle: Projection, Not Storage - -Lith-Geo does NOT store source data. It: - -1. Reads documents from Lith via HTTP API -2. Extracts location fields (lat/lon, GeoJSON, WKT) -3. Builds spatial index as materialized view -4. Serves spatial queries against the index -5. Returns Lith document IDs for full provenance - -[source,text] ----- -Lith Journal (source of truth) - │ - ├── doc_123: { location: { lat: 51.5, lon: -0.1 }, ... } - ├── doc_456: { location: { lat: 48.8, lon: 2.3 }, ... } - │ - ▼ -Lith-Geo R-tree Index (derived) - │ - ├── Point(51.5, -0.1) → doc_123 - ├── Point(48.8, 2.3) → doc_456 - │ - ▼ -Spatial Query: "within 10km of London" - │ - ▼ -Result: [doc_123] (with full Lith provenance) ----- - -== Features - -=== Spatial Query Types - -* **Point queries**: Find documents at exact coordinates -* **Bounding box**: Documents within rectangular region -* **Radius search**: Documents within distance of point -* **Polygon containment**: Documents within arbitrary shape -* **Nearest neighbor**: K closest documents to point - -=== Supported Geometry Types - -* Point (lat/lon) -* LineString -* Polygon -* MultiPoint, MultiLineString, MultiPolygon -* GeometryCollection - -=== Coordinate Systems - -* WGS84 (EPSG:4326) - default -* Web Mercator (EPSG:3857) -* Custom CRS via PROJ - -== API - -=== Endpoints - -[source,text] ----- -GET /geo/within-bbox?minLat=...&minLon=...&maxLat=...&maxLon=... -GET /geo/within-radius?lat=...&lon=...&radius=...&unit=km -GET /geo/within-polygon?geojson=... -GET /geo/nearest?lat=...&lon=...&k=10 -POST /geo/reindex # Rebuild from Lith -GET /geo/health # Index status ----- - -=== Example Queries - -[source,bash] ----- -# Find evidence within 50km of London -curl "http://localhost:8081/geo/within-radius?lat=51.5074&lon=-0.1278&radius=50&unit=km" - -# Find evidence in bounding box (UK) -curl "http://localhost:8081/geo/within-bbox?minLat=49.9&minLon=-6.4&maxLat=55.8&maxLon=1.8" - -# Find 10 nearest documents to a location -curl "http://localhost:8081/geo/nearest?lat=48.8566&lon=2.3522&k=10" ----- - -=== Response Format - -[source,json] ----- -{ - "query": { - "type": "within-radius", - "center": { "lat": 51.5074, "lon": -0.1278 }, - "radius_km": 50 - }, - "results": [ - { - "lithoglyph_id": "doc_123", - "location": { "lat": 51.5, "lon": -0.1 }, - "distance_km": 0.82, - "provenance_url": "http://lithoglyph:8080/documents/doc_123" - } - ], - "index_timestamp": "2026-01-16T12:00:00Z", - "total_indexed": 15234 -} ----- - -== Installation - -=== Prerequisites - -* Rust 1.75+ (stable) -* Lith running with HTTP API enabled - -=== Build - -[source,bash] ----- -git clone https://github.com/hyperpolymath/lithoglyph-geo -cd lithoglyph-geo -cargo build --release ----- - -=== Configuration - -[source,toml] ----- -# lithoglyph-geo.toml -[lithoglyph] -api_url = "http://localhost:8080" -collection = "evidence" # Collection to index -location_field = "location" # Field containing coordinates - -[server] -host = "127.0.0.1" -port = 8081 - -[index] -# Rebuild interval (0 = manual only) -auto_rebuild_minutes = 0 -# Memory limit for R-tree -max_memory_mb = 512 ----- - -=== Run - -[source,bash] ----- -# Start the server -./target/release/lithoglyph-geo --config lithoglyph-geo.toml - -# Build initial index -curl -X POST http://localhost:8081/geo/reindex ----- - -== Technology Stack - -|=== -| Component | Technology | Rationale - -| Core -| Rust -| Performance, memory safety, no GC pauses - -| Spatial Index -| rstar (R-tree) -| Pure Rust, no C dependencies - -| HTTP Server -| axum -| Async, tower ecosystem - -| Geometry -| geo crate -| Rust-native geometry primitives - -| Serialization -| serde + GeoJSON -| Standard formats -|=== - -== Non-Goals - -* **Not a PostGIS replacement** - Use PostGIS for heavy GIS workloads -* **Not a tile server** - Use Martin/pg_tileserv for map tiles -* **Not storing source data** - Lith is the source of truth -* **Not real-time tracking** - Batch indexing from Lith journal - -== Project Structure - -[source,text] ----- -lithoglyph-geo/ -├── src/ -│ ├── main.rs # Entry point -│ ├── lib.rs # Library root -│ ├── index/ -│ │ ├── mod.rs -│ │ ├── rtree.rs # R-tree implementation -│ │ └── projection.rs # Lith → spatial projection -│ ├── api/ -│ │ ├── mod.rs -│ │ ├── routes.rs # HTTP endpoints -│ │ └── queries.rs # Spatial query handlers -│ ├── lithoglyph/ -│ │ ├── mod.rs -│ │ └── client.rs # Lith HTTP client -│ └── config.rs # Configuration -├── spec/ -│ └── spatial-queries.adoc # Query specification -├── Cargo.toml -├── README.adoc -├── ECOSYSTEM.scm -├── STATE.scm -├── META.scm -└── LICENSE ----- - -== Roadmap - -=== v0.1.0: Core Indexing (MVP) -* [ ] R-tree spatial index -* [ ] Lith HTTP client -* [ ] Bounding box queries -* [ ] Radius queries -* [ ] Manual reindex endpoint - -=== v0.2.0: Advanced Queries -* [ ] Polygon containment -* [ ] Nearest neighbor (k-NN) -* [ ] LineString queries (route intersection) - -=== v0.3.0: Performance -* [ ] Incremental index updates (watch Lith journal) -* [ ] Index persistence (avoid full rebuild on restart) -* [ ] Query result caching - -=== v0.4.0: Integration -* [ ] Lith Studio integration (map view) -* [ ] GeoJSON export -* [ ] Bulk import from external GIS - -== Related Projects - -* https://github.com/hyperpolymath/lithoglyph[Lith] - The narrative-first database (source of truth) -* https://github.com/hyperpolymath/lithoglyph-studio[Lith Studio] - GUI (map visualization target) -* https://github.com/hyperpolymath/lithoglyph-analytics[Lith Analytics] - OLAP analytics engine -* https://github.com/hyperpolymath/bofig[BoFIG] - Evidence graph (potential consumer) - -== License - -PMPL-1.0-or-later diff --git a/lithoglyph/geo/ROADMAP.adoc b/lithoglyph/geo/ROADMAP.adoc deleted file mode 100644 index a9098424..00000000 --- a/lithoglyph/geo/ROADMAP.adoc +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Geo Roadmap - -== Current Status - -Initial development phase. - -== Milestones - -=== v0.1.0 - Foundation -* [ ] Core functionality -* [ ] Basic documentation -* [ ] CI/CD pipeline - -=== v1.0.0 - Stable Release -* [ ] Full feature set -* [ ] Comprehensive tests -* [ ] Production ready - -== Future Directions - -_To be determined based on community feedback._ diff --git a/lithoglyph/geo/RSR_OUTLINE.adoc b/lithoglyph/geo/RSR_OUTLINE.adoc deleted file mode 100644 index ede631fe..00000000 --- a/lithoglyph/geo/RSR_OUTLINE.adoc +++ /dev/null @@ -1,218 +0,0 @@ -= RSR Template Repository - -image:[Palimpsest-MPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] image:[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] -:toc: -:sectnums: - -// Badges -image:https://img.shields.io/badge/RSR-Infrastructure-cd7f32[RSR Infrastructure] -image:https://img.shields.io/badge/Phase-Maintenance-brightgreen[Phase] -image:https://img.shields.io/badge/Guix-Primary-purple?logo=gnu[Guix] - -== Overview - -**The canonical template for RSR (Rhodium Standard Repository) projects.** - -This repository provides the standardized structure, configuration, and tooling for all 139 repos in the hyperpolymath ecosystem. Use it to: - -* Bootstrap new projects with RSR compliance -* Reference the standard directory structure -* Copy configuration templates (Justfile, STATE.scm, etc.) - -== Quick Start - -[source,bash] ----- -# Clone the template -git clone https://github.com/hyperpolymath/RSR-template-repo my-project -cd my-project - -# Remove template git history -rm -rf .git -git init - -# Customize -sed -i 's/RSR-template-repo/my-project/g' Justfile guix.scm README.adoc - -# Enter development environment -guix shell -D -f guix.scm - -# Validate compliance -just validate-rsr ----- - -== What's Included - -[cols="1,3"] -|=== -|File/Directory |Purpose - -|`.editorconfig` -|Editor configuration (indent, charset) - -|`.gitignore` -|Standard ignore patterns - -|`.guix-channel` -|Guix channel definition - -|`.well-known/` -|RFC-compliant metadata (security.txt, ai.txt, humans.txt) - -|`docs/` -|Documentation directory - -|`guix.scm` -|Guix package definition - -|`justfile` -|Task runner with 50+ recipes - -|`LICENSE.txt` -|Palimpsest (PMPL-1.0-or-later) - -|`README.adoc` -|This file - -|`RSR_COMPLIANCE.adoc` -|Compliance tracking - -|`STATE.scm` -|Project state checkpoint -|=== - -== Justfile Features - -The template Justfile provides: - -* **~10 billion recipe combinations** via matrix recipes -* **Cookbook generation**: `just cookbook` → `docs/just-cookbook.adoc` -* **Man page generation**: `just man` → `docs/man/project.1` -* **RSR validation**: `just validate-rsr` -* **STATE.scm management**: `just state-touch`, `just state-phase` -* **Container support**: `just container-build`, `just container-push` -* **CI matrix**: `just ci-matrix [stage] [depth]` - -=== Key Recipes - -[source,bash] ----- -just # Show all recipes -just help # Detailed help -just info # Project info -just combinations # Show matrix options - -just build # Build (debug) -just test # Run tests -just quality # Format + lint + test -just ci # Full CI pipeline - -just validate # RSR + STATE validation -just docs # Generate all docs -just cookbook # Generate Justfile docs - -just guix-shell # Guix dev environment -just container-build # Build container ----- - -== Directory Structure - -[source] ----- -project/ -├── .editorconfig # Editor settings -├── .gitignore # Git ignore -├── .guix-channel # Guix channel -├── .well-known/ # RFC metadata -│ ├── ai.txt -│ ├── humans.txt -│ └── security.txt -├── config/ # Nickel configs (optional) -├── docs/ # Documentation -│ ├── generated/ -│ ├── man/ -│ └── just-cookbook.adoc -├── guix.scm # Guix package -├── Justfile # Task runner -├── LICENSE.txt # Dual license -├── README.adoc # Overview -├── RSR_COMPLIANCE.adoc # Compliance -├── src/ # Source code -├── STATE.scm # State checkpoint -└── tests/ # Tests ----- - -== RSR Compliance - -=== Language Tiers - -* **Tier 1** (Gold): Rust, Elixir, Zig, Ada, Haskell, ReScript -* **Tier 2** (Silver): Nickel, Racket, Guile Scheme, Nix -* **Infrastructure**: Guix channels, derivations - -=== Required Files - -* `.editorconfig` -* `.gitignore` -* `justfile` -* `README.adoc` -* `RSR_COMPLIANCE.adoc` -* `LICENSE` (PMPL-1.0-or-later) -* `.well-known/security.txt` -* `.well-known/ai.txt` -* `.well-known/humans.txt` -* `guix.scm` OR `flake.nix` - -=== Prohibited - -* Python outside `salt/` directory -* TypeScript/JavaScript (use ReScript) -* CUE (use Guile/Nickel) -* `Dockerfile` (use `Containerfile`) - -== STATE.scm - -The STATE.scm file tracks project state: - -[source,scheme] ----- -(define state - `((metadata - (project . "my-project") - (updated . "2025-12-10")) - (position - (phase . implementation) ; design|implementation|testing|maintenance|archived - (maturity . beta)) ; experimental|alpha|beta|production|lts - (ecosystem - (part-of . ("RSR Framework")) - (depends-on . ())))) ----- - -== Badge Schema - -Generate badges from STATE.scm: - -[source,bash] ----- -just badges standard ----- - -See `docs/BADGE_SCHEMA.adoc` for the full badge taxonomy. - -== Ecosystem Integration - -This template is part of: - -* **STATE.scm Ecosystem**: Conversation checkpoints -* **RSR Framework**: Repository standards -* **Consent-Aware-HTTP**: .well-known compliance - -== License - -SPDX-License-Identifier: CC-BY-SA-4.0 - -== Links - -* https://github.com/hyperpolymath/elegant-STATE[elegant-STATE] - STATE.scm tooling -* https://github.com/hyperpolymath/conative-gating[conative-gating] - Policy enforcement -* https://rhodium.sh[Rhodium Standard] - RSR documentation diff --git a/lithoglyph/geo/UNIFIED-ROADMAP.scm b/lithoglyph/geo/UNIFIED-ROADMAP.scm deleted file mode 100644 index 3e5b6efe..00000000 --- a/lithoglyph/geo/UNIFIED-ROADMAP.scm +++ /dev/null @@ -1,248 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; Lith Ecosystem - Unified Roadmap to MVP 1.0.0 -; Media-Type: application/vnd.roadmap+scm -; -; This file is distributed to all Lith ecosystem repos: -; - lith (core database) -; - gql-dt (dependently-typed query language) -; - lith-studio (GUI) -; - lith-debugger (recovery tool) -; - lith-geo (geospatial projection layer) -; - lith-analytics (OLAP analytics projection layer) - -(unified-roadmap - (metadata - (version "1.1.0") - (created "2026-01-12") - (updated "2026-01-16") - (author "hyperpolymath") - (target "MVP 1.0.0")) - - ;; ============================================================================ - ;; ECOSYSTEM OVERVIEW - ;; ============================================================================ - (ecosystem-summary - (components - (lith - (version "0.0.5") - (completion 80) - (role "Core database engine") - (tech "Forth + Factor + Zig")) - (gql-dt - (version "0.2.0") - (completion 65) - (role "Dependently-typed query language") - (tech "Lean 4 + Zig")) - (lith-studio - (version "0.1.0") - (completion 45) - (role "Zero-friction GUI") - (tech "ReScript + Tauri 2.0 + Rust")) - (lith-debugger - (version "0.1.0") - (completion 55) - (role "Proof-carrying recovery tool") - (tech "Lean 4 + Idris 2 + Rust")) - (lith-geo - (version "0.1.0") - (completion 15) - (role "Geospatial projection layer") - (tech "Rust (rstar, axum, geo)")) - (lith-analytics - (version "0.1.0") - (completion 15) - (role "OLAP analytics projection layer") - (tech "Julia (DataFrames, Parquet2, Oxygen)"))) - - (architecture - "┌─────────────────────────────────────────────────────────────┐" - "│ Lith Studio (GUI) │" - "│ ↓ generates FQLdt code │" - "├─────────────────────────────────────────────────────────────┤" - "│ FQLdt (Lean 4) │" - "│ ↓ compiles to proof blobs │" - "├─────────────────────────────────────────────────────────────┤" - "│ Form.Bridge (Zig ABI) │" - "│ ↓ calls │" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Core (Forth + Factor) │" - "│ Form.Runtime → Form.Normalizer → Form.Model → Form.Blocks│" - "├─────────────────────────────────────────────────────────────┤" - "│ Lith Debugger (alongside) │" - "│ ↓ proves recovery safe │" - "│ Lith + FQLdt │" - "├─────────────────────────────────────────────────────────────┤" - "│ Projection Layers (read from Lith HTTP API) │" - "│ lith-geo (R-tree spatial) lith-analytics (OLAP) │" - "└─────────────────────────────────────────────────────────────┘")) - - ;; ============================================================================ - ;; CRITICAL PATH TO MVP 1.0.0 - ;; ============================================================================ - (critical-path - (phase (id "P1") (name "Core Integration") - (duration "weeks 1-6") - (focus "Lith + FQLdt integration") - - (lith-tasks - (task "Complete M11: HTTP API Server" priority: critical status: complete) - (task "M12: Language bindings (ReScript, PHP)" priority: critical status: next) - (task "M13: CMS integration (WordPress)" priority: high status: pending) - (task "M14: Form.ControlPlane (clustering)" priority: medium status: pending)) - - (gql-dt-tasks - (task "M5: Zig FFI bridge to Form.Bridge" priority: critical status: not-started) - (task "M6: GQL parser (integrate with Lith's EBNF)" priority: high status: not-started) - (task "Proof blob serialization (CBOR RFC 8949)" priority: high status: pending)) - - (checkpoint "FQLdt can compile a query → proof blob → Lith accepts and executes")) - - (phase (id "P2") (name "User-Facing Tools") - (duration "weeks 7-10") - (focus "Studio and Debugger completion") - - (studio-tasks - (task "Verify ReScript/Tauri build pipeline" priority: critical status: pending) - (task "Wire ReScript UI to FQLdt code generation" priority: high status: pending) - (task "Connect to Lith HTTP API" priority: high status: blocked) - (task "Test schema creation → query → results flow" priority: medium status: pending)) - - (debugger-tasks - (task "Wire Idris REPL to PostgreSQL adapter" priority: high status: pending) - (task "Lith adapter: parse real journal files" priority: high status: partial) - (task "Complete Ratatui TUI interface" priority: medium status: in-progress) - (task "Integration: proof verification before recovery" priority: medium status: pending)) - - (projection-layer-tasks - (task "lith-geo: Integration test with real Lith" priority: high status: pending) - (task "lith-geo: Docker deployment" priority: medium status: pending) - (task "lith-analytics: Integration test with real Lith" priority: high status: pending) - (task "lith-analytics: PROMPT score dashboard endpoints" priority: medium status: pending)) - - (checkpoint "Users can create schemas in Studio, debug with Debugger, query spatial/analytics")) - - (phase (id "P3") (name "Production Hardening") - (duration "weeks 11-12") - (focus "Stability and polish") - - (all-repos - (task "Crash recovery tests" priority: high) - (task "Error handling improvements" priority: high) - (task "Cross-platform testing" priority: medium) - (task "Documentation completion" priority: medium) - (task "Performance optimization" priority: low)) - - (checkpoint "MVP 1.0.0 release ready"))) - - ;; ============================================================================ - ;; DEPENDENCY GRAPH - ;; ============================================================================ - (dependencies - (lith-m11 - (name "Lith HTTP API Server") - (blocks "Studio M2" "Debugger Lith adapter") - (priority critical)) - - (gql-dt-m5 - (name "FQLdt Zig FFI Bridge") - (blocks "Studio M3" "Real type checking") - (depends-on "Lith Form.Bridge") - (priority critical)) - - (gql-dt-m6 - (name "FQLdt GQL Parser") - (blocks "Full FQLdt compilation") - (depends-on "gql-dt-m5") - (priority high)) - - (studio-m1 - (name "Studio Build Pipeline") - (blocks "All Studio features") - (priority critical)) - - (debugger-repl-db - (name "Debugger REPL Database Connection") - (blocks "Real debugging") - (priority high)) - - (lith-geo-integration - (name "lith-geo Lith Integration") - (depends-on "lith-m11") - (blocks "Spatial queries in Studio") - (priority medium)) - - (lith-analytics-integration - (name "lith-analytics Lith Integration") - (depends-on "lith-m11") - (blocks "Analytics dashboards in Studio") - (priority medium))) - - ;; ============================================================================ - ;; UNRESOLVED DECISIONS - ;; ============================================================================ - (decisions-needed - (decision (id "DECISION-002") - (title "FQLdt parser approach") - (repo "gql-dt") - (options - "Hand-rolled parser (simple, no deps)" - "Lean 4 Parsec (built-in)" - "Integrate with Lith's Factor-based GQL parser") - (recommendation "Integrate - reuse Lith's EBNF grammar via FFI") - (impact "Affects M6 implementation")) - - (decision (id "DECISION-003") - (title "Lith integration strategy for FQLdt") - (repo "gql-dt") - (options - "Mock Forth core for MVP" - "Real Form.Bridge integration") - (recommendation "Real integration - M11 HTTP API makes this feasible") - (impact "Determines MVP scope"))) - - ;; ============================================================================ - ;; POST-MVP ROADMAP - ;; ============================================================================ - (post-mvp - (release (version "1.1.0") (name "Normalization & Migration") - (features - "Form.Normalizer full integration (FD discovery → decomposition)" - "Three-phase migration workflow (Announce/Shadow/Commit)" - "Studio: visual normalization wizard" - "Debugger: migration rollback proofs" - "lith-geo: Polygon and region queries" - "lith-analytics: Time-series dashboards in Studio")) - - (release (version "1.2.0") (name "Multi-Database Support") - (features - "Debugger: SQLite adapter completion" - "Lith: clustering/replication (Form.ControlPlane begins)" - "Studio: connection manager for multiple DBs")) - - (release (version "2.0.0") (name "Agentic Ecosystem") - (features - "Form.ControlPlane (Elixir/OTP) for distributed coordination" - "Agent handover protocols" - "Long-term archive format standardization" - "Multi-user collaboration in Studio"))) - - ;; ============================================================================ - ;; SUCCESS METRICS - ;; ============================================================================ - (success-metrics - (mvp-criteria - "User can create a schema in Studio with visual builder" - "Schema generates valid FQLdt with type checking" - "User can insert data with provenance tracking" - "User can query data and see results" - "Debugger can analyze schema and propose fixes" - "All operations have proof-carrying verification") - - (quality-gates - "All ReScript code compiles without warnings" - "All Rust code passes Clippy lints" - "All Lean 4 code builds with lake" - "All Julia code passes tests" - "Cross-platform builds succeed (Mac/Windows/Linux)" - "Integration tests pass end-to-end" - "Projection layers can sync from Lith HTTP API"))) diff --git a/lithoglyph/geo/config.example.toml b/lithoglyph/geo/config.example.toml deleted file mode 100644 index 92a807b7..00000000 --- a/lithoglyph/geo/config.example.toml +++ /dev/null @@ -1,22 +0,0 @@ -# Lith-Geo Configuration Example -# Copy to config.toml and adjust as needed - -[lithoglyph] -# Lith HTTP API URL -api_url = "http://localhost:8080" -# Collection containing documents with location data -collection = "evidence" -# Field name containing location coordinates -location_field = "location" - -[server] -# Host to bind to -host = "127.0.0.1" -# Port to listen on -port = 8081 - -[index] -# Auto-rebuild interval in minutes (0 = manual only via POST /geo/reindex) -auto_rebuild_minutes = 0 -# Maximum memory for R-tree index in MB -max_memory_mb = 512 diff --git a/lithoglyph/geo/docs/CITATIONS.adoc b/lithoglyph/geo/docs/CITATIONS.adoc deleted file mode 100644 index 6f167bdf..00000000 --- a/lithoglyph/geo/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {PMPL-1.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/lithoglyph/geo/examples/SafeDOMExample.affine b/lithoglyph/geo/examples/SafeDOMExample.affine deleted file mode 100644 index 2a62c1d1..00000000 --- a/lithoglyph/geo/examples/SafeDOMExample.affine +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SafeDOMExample.affine — formally-verified DOM mounting (aspirational). -// -// This example shows the *shape* of SafeDOM consumer code in current -// AffineScript syntax. The `SafeDOM` stdlib surface it references -// (`mount_safe`, `mount_when_ready`, `mount_batch`, -// `proven_selector_validate`, `proven_html_validate`, `mount`) is the -// target of `affinescript#56` (DOM+Pixi binding survey) and does not -// yet exist in the published stdlib. The file is therefore -// parse-checked but not type-checked end-to-end until #56 lands the -// bindings; `affinescript check` reports `Resolve.UndefinedModule -// SafeDOM` which is expected. -// -// Previous versions of this file (estate-wide, 5 dialect variants) -// pre-dated ADR-014 (qualified paths), ADR-016 (effect rows), and the -// `#{`-record-literal sigil (ADR-215). They were retired in favour of -// this canonical via the gitbot-fleet#208 sweep (2026-05-26). - -module SafeDOMExample; - -use prelude::{Option, Some, None, Result, Ok, Err}; - -// `Element` and friends are nominal extern types for now — the real -// shape lands with affinescript#56. -extern type Element; -extern type Selector; -extern type ValidHTML; - -// Single-mount status, lifted from the host into a typed tag union. -enum MountStatus { - Mounted(Element), - MountPointNotFound(String), - InvalidSelector(String), - InvalidHTML(String) -} - -// Batch-mount result. -enum MountResult { - Mounted([Element]), - Failed(String) -} - -// Spec for one element in a batch mount. -struct MountSpec { - selector: String, - html: String -} - -// SafeDOM's host-side surface, all IO-effecting. Callbacks are passed -// as separate parameters (rather than a `MountCallbacks` record) -// because fn-typed struct fields are not currently parser-supported. -extern fn mount_safe( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_when_ready( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_batch(specs: ref [MountSpec]) -{IO}-> MountResult; - -extern fn proven_selector_validate(s: ref String) -{IO}-> Result; -extern fn proven_html_validate(s: ref String) -{IO}-> Result; -extern fn mount(sel: ref Selector, html: ref ValidHTML) -{IO}-> MountStatus; - -extern fn array_for_each(xs: ref [Element], f: fn(Element) -> ()) -{IO}-> (); -extern fn array_len(xs: ref [Element]) -> Int; - -// Example 1 — basic mount with success/error branches. -pub fn mount_app() -{IO}-> () { - mount_safe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - fn(el) -> () { Console::log("App mounted successfully"); }, - fn(err) -> () { Console::error("Mount failed: " ++ err); }, - ); -} - -// Example 2 — defer until DOM ready. -pub fn mount_when_dom_ready() -{IO}-> () { - mount_when_ready( - "#app", - "

App Title

", - fn(_el) -> () { Console::log("Mounted after DOM ready"); }, - fn(err) -> () { Console::error("Failed: " ++ err); }, - ); -} - -// Example 3 — atomic batch mount. -pub fn mount_multiple() -{IO}-> () { - let specs = [ - MountSpec #{ selector: "#header", html: "

Site Title

" }, - MountSpec #{ selector: "#nav", html: "" }, - MountSpec #{ selector: "#main", html: "

Content here

" }, - MountSpec #{ selector: "#footer", html: "
2026
" }, - ]; - - match mount_batch(specs) { - Mounted(elements) => { - Console::log("Batch mount succeeded"); - array_for_each(elements, fn(_el) -> () { Console::log(" element"); }); - }, - Failed(err) => { - Console::error("Batch mount failed (atomic — none mounted): " ++ err); - } - } -} - -// Example 4 — explicit two-stage validation before mounting. -pub fn mount_with_validation() -{IO}-> () { - match proven_selector_validate("#my-app") { - Err(e) => Console::error("Invalid selector: " ++ e), - Ok(valid_selector) => match proven_html_validate("
Content
") { - Err(e) => Console::error("Invalid HTML: " ++ e), - Ok(valid_html) => match mount(valid_selector, valid_html) { - Mounted(_el) => Console::log("Mounted with validated inputs"), - MountPointNotFound(s) => Console::error("Element not found: " ++ s), - InvalidSelector(_) => Console::error("impossible — already validated"), - InvalidHTML(_) => Console::error("impossible — already validated"), - }, - }, - } -} diff --git a/lithoglyph/geo/examples/web-project-deno.json b/lithoglyph/geo/examples/web-project-deno.json deleted file mode 100644 index 5ddd3bd7..00000000 --- a/lithoglyph/geo/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "^12.0.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/lithoglyph/geo/ffi/zig/build.zig b/lithoglyph/geo/ffi/zig/build.zig deleted file mode 100644 index 4a2e049a..00000000 --- a/lithoglyph/geo/ffi/zig/build.zig +++ /dev/null @@ -1,94 +0,0 @@ -// {{PROJECT}} FFI Build Configuration -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Shared library (.so, .dylib, .dll) - const lib = b.addSharedLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Set version - lib.version = .{ .major = 0, .minor = 1, .patch = 0 }; - - // Static library (.a) - const lib_static = b.addStaticLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Install artifacts - b.installArtifact(lib); - b.installArtifact(lib_static); - - // Generate header file for C compatibility - const header = b.addInstallHeader( - b.path("include/{{project}}.h"), - "{{project}}.h", - ); - b.getInstallStep().dependOn(&header.step); - - // Unit tests - const lib_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const run_lib_tests = b.addRunArtifact(lib_tests); - - const test_step = b.step("test", "Run library tests"); - test_step.dependOn(&run_lib_tests.step); - - // Integration tests - const integration_tests = b.addTest(.{ - .root_source_file = b.path("test/integration_test.zig"), - .target = target, - .optimize = optimize, - }); - - integration_tests.linkLibrary(lib); - - const run_integration_tests = b.addRunArtifact(integration_tests); - - const integration_test_step = b.step("test-integration", "Run integration tests"); - integration_test_step.dependOn(&run_integration_tests.step); - - // Documentation - const docs = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = .Debug, - }); - - const docs_step = b.step("docs", "Generate documentation"); - docs_step.dependOn(&b.addInstallDirectory(.{ - .source_dir = docs.getEmittedDocs(), - .install_dir = .prefix, - .install_subdir = "docs", - }).step); - - // Benchmark (if needed) - const bench = b.addExecutable(.{ - .name = "{{project}}-bench", - .root_source_file = b.path("bench/bench.zig"), - .target = target, - .optimize = .ReleaseFast, - }); - - bench.linkLibrary(lib); - - const run_bench = b.addRunArtifact(bench); - - const bench_step = b.step("bench", "Run benchmarks"); - bench_step.dependOn(&run_bench.step); -} diff --git a/lithoglyph/geo/ffi/zig/src/main.zig b/lithoglyph/geo/ffi/zig/src/main.zig deleted file mode 100644 index 6b233bc7..00000000 --- a/lithoglyph/geo/ffi/zig/src/main.zig +++ /dev/null @@ -1,274 +0,0 @@ -// {{PROJECT}} FFI Implementation -// -// This module implements the C-compatible FFI declared in src/abi/Foreign.idr -// All types and layouts must match the Idris2 ABI definitions. -// -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -// Version information (keep in sync with project) -const VERSION = "0.1.0"; -const BUILD_INFO = "{{PROJECT}} built with Zig " ++ @import("builtin").zig_version_string; - -/// Thread-local error storage -threadlocal var last_error: ?[]const u8 = null; - -/// Set the last error message -fn setError(msg: []const u8) void { - last_error = msg; -} - -/// Clear the last error -fn clearError() void { - last_error = null; -} - -//============================================================================== -// Core Types (must match src/abi/Types.idr) -//============================================================================== - -/// Result codes (must match Idris2 Result type) -pub const Result = enum(c_int) { - ok = 0, - @"error" = 1, - invalid_param = 2, - out_of_memory = 3, - null_pointer = 4, -}; - -/// Library handle (opaque to prevent direct access) -pub const Handle = opaque { - // Internal state hidden from C - allocator: std.mem.Allocator, - initialized: bool, - // Add your fields here -}; - -//============================================================================== -// Library Lifecycle -//============================================================================== - -/// Initialize the library -/// Returns a handle, or null on failure -export fn {{project}}_init() ?*Handle { - const allocator = std.heap.c_allocator; - - const handle = allocator.create(Handle) catch { - setError("Failed to allocate handle"); - return null; - }; - - // Initialize handle - handle.* = .{ - .allocator = allocator, - .initialized = true, - }; - - clearError(); - return handle; -} - -/// Free the library handle -export fn {{project}}_free(handle: ?*Handle) void { - const h = handle orelse return; - const allocator = h.allocator; - - // Clean up resources - h.initialized = false; - - allocator.destroy(h); - clearError(); -} - -//============================================================================== -// Core Operations -//============================================================================== - -/// Process data (example operation) -export fn {{project}}_process(handle: ?*Handle, input: u32) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Example processing logic - _ = input; - - clearError(); - return .ok; -} - -//============================================================================== -// String Operations -//============================================================================== - -/// Get a string result (example) -/// Caller must free the returned string -export fn {{project}}_get_string(handle: ?*Handle) ?[*:0]const u8 { - const h = handle orelse { - setError("Null handle"); - return null; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return null; - } - - // Example: allocate and return a string - const result = h.allocator.dupeZ(u8, "Example result") catch { - setError("Failed to allocate string"); - return null; - }; - - clearError(); - return result.ptr; -} - -/// Free a string allocated by the library -export fn {{project}}_free_string(str: ?[*:0]const u8) void { - const s = str orelse return; - const allocator = std.heap.c_allocator; - - const slice = std.mem.span(s); - allocator.free(slice); -} - -//============================================================================== -// Array/Buffer Operations -//============================================================================== - -/// Process an array of data -export fn {{project}}_process_array( - handle: ?*Handle, - buffer: ?[*]const u8, - len: u32, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const buf = buffer orelse { - setError("Null buffer"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Access the buffer - const data = buf[0..len]; - _ = data; - - // Process data here - - clearError(); - return .ok; -} - -//============================================================================== -// Error Handling -//============================================================================== - -/// Get the last error message -/// Returns null if no error -export fn {{project}}_last_error() ?[*:0]const u8 { - const err = last_error orelse return null; - - // Return C string (static storage, no need to free) - const allocator = std.heap.c_allocator; - const c_str = allocator.dupeZ(u8, err) catch return null; - return c_str.ptr; -} - -//============================================================================== -// Version Information -//============================================================================== - -/// Get the library version -export fn {{project}}_version() [*:0]const u8 { - return VERSION.ptr; -} - -/// Get build information -export fn {{project}}_build_info() [*:0]const u8 { - return BUILD_INFO.ptr; -} - -//============================================================================== -// Callback Support -//============================================================================== - -/// Callback function type (C ABI) -pub const Callback = *const fn (u64, u32) callconv(.C) u32; - -/// Register a callback -export fn {{project}}_register_callback( - handle: ?*Handle, - callback: ?Callback, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const cb = callback orelse { - setError("Null callback"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Store callback for later use - _ = cb; - - clearError(); - return .ok; -} - -//============================================================================== -// Utility Functions -//============================================================================== - -/// Check if handle is initialized -export fn {{project}}_is_initialized(handle: ?*Handle) u32 { - const h = handle orelse return 0; - return if (h.initialized) 1 else 0; -} - -//============================================================================== -// Tests -//============================================================================== - -test "lifecycle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try std.testing.expect({{project}}_is_initialized(handle) == 1); -} - -test "error handling" { - const result = {{project}}_process(null, 0); - try std.testing.expectEqual(Result.null_pointer, result); - - const err = {{project}}_last_error(); - try std.testing.expect(err != null); -} - -test "version" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - try std.testing.expectEqualStrings(VERSION, ver_str); -} diff --git a/lithoglyph/geo/ffi/zig/test/integration_test.zig b/lithoglyph/geo/ffi/zig/test/integration_test.zig deleted file mode 100644 index 03419949..00000000 --- a/lithoglyph/geo/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,182 +0,0 @@ -// {{PROJECT}} Integration Tests -// SPDX-License-Identifier: MPL-2.0 -// -// These tests verify that the Zig FFI correctly implements the Idris2 ABI - -const std = @import("std"); -const testing = std.testing; - -// Import FFI functions -extern fn {{project}}_init() ?*opaque {}; -extern fn {{project}}_free(?*opaque {}) void; -extern fn {{project}}_process(?*opaque {}, u32) c_int; -extern fn {{project}}_get_string(?*opaque {}) ?[*:0]const u8; -extern fn {{project}}_free_string(?[*:0]const u8) void; -extern fn {{project}}_last_error() ?[*:0]const u8; -extern fn {{project}}_version() [*:0]const u8; -extern fn {{project}}_is_initialized(?*opaque {}) u32; - -//============================================================================== -// Lifecycle Tests -//============================================================================== - -test "create and destroy handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try testing.expect(handle != null); -} - -test "handle is initialized" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const initialized = {{project}}_is_initialized(handle); - try testing.expectEqual(@as(u32, 1), initialized); -} - -test "null handle is not initialized" { - const initialized = {{project}}_is_initialized(null); - try testing.expectEqual(@as(u32, 0), initialized); -} - -//============================================================================== -// Operation Tests -//============================================================================== - -test "process with valid handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const result = {{project}}_process(handle, 42); - try testing.expectEqual(@as(c_int, 0), result); // 0 = ok -} - -test "process with null handle returns error" { - const result = {{project}}_process(null, 42); - try testing.expectEqual(@as(c_int, 4), result); // 4 = null_pointer -} - -//============================================================================== -// String Tests -//============================================================================== - -test "get string result" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const str = {{project}}_get_string(handle); - defer if (str) |s| {{project}}_free_string(s); - - try testing.expect(str != null); -} - -test "get string with null handle" { - const str = {{project}}_get_string(null); - try testing.expect(str == null); -} - -//============================================================================== -// Error Handling Tests -//============================================================================== - -test "last error after null handle operation" { - _ = {{project}}_process(null, 0); - - const err = {{project}}_last_error(); - try testing.expect(err != null); - - if (err) |e| { - const err_str = std.mem.span(e); - try testing.expect(err_str.len > 0); - } -} - -test "no error after successful operation" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - _ = {{project}}_process(handle, 0); - - // Error should be cleared after successful operation - // (This depends on implementation) -} - -//============================================================================== -// Version Tests -//============================================================================== - -test "version string is not empty" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - try testing.expect(ver_str.len > 0); -} - -test "version string is semantic version format" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - // Should be in format X.Y.Z - try testing.expect(std.mem.count(u8, ver_str, ".") >= 1); -} - -//============================================================================== -// Memory Safety Tests -//============================================================================== - -test "multiple handles are independent" { - const h1 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h1); - - const h2 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h2); - - try testing.expect(h1 != h2); - - // Operations on h1 should not affect h2 - _ = {{project}}_process(h1, 1); - _ = {{project}}_process(h2, 2); -} - -test "double free is safe" { - const handle = {{project}}_init() orelse return error.InitFailed; - - {{project}}_free(handle); - {{project}}_free(handle); // Should not crash -} - -test "free null is safe" { - {{project}}_free(null); // Should not crash -} - -//============================================================================== -// Thread Safety Tests (if applicable) -//============================================================================== - -test "concurrent operations" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const ThreadContext = struct { - h: *opaque {}, - id: u32, - }; - - const thread_fn = struct { - fn run(ctx: ThreadContext) void { - _ = {{project}}_process(ctx.h, ctx.id); - } - }.run; - - var threads: [4]std.Thread = undefined; - for (&threads, 0..) |*thread, i| { - thread.* = try std.Thread.spawn(.{}, thread_fn, .{ - ThreadContext{ .h = handle, .id = @intCast(i) }, - }); - } - - for (threads) |thread| { - thread.join(); - } -} diff --git a/lithoglyph/geo/fuzz/Cargo.toml b/lithoglyph/geo/fuzz/Cargo.toml deleted file mode 100644 index dc5f44ac..00000000 --- a/lithoglyph/geo/fuzz/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "fuzz" -version = "0.0.0" -publish = false -edition = "2021" - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.4" - -[dependencies.lithoglyph-geo] -path = ".." - -[[bin]] -name = "fuzz_main" -path = "fuzz_targets/fuzz_main.rs" -test = false -doc = false diff --git a/lithoglyph/geo/fuzz/fuzz_targets/fuzz_main.rs b/lithoglyph/geo/fuzz/fuzz_targets/fuzz_main.rs deleted file mode 100644 index 1f71ba3b..00000000 --- a/lithoglyph/geo/fuzz/fuzz_targets/fuzz_main.rs +++ /dev/null @@ -1,22 +0,0 @@ -#![no_main] -use libfuzzer_sys::fuzz_target; - -fuzz_target!(|data: &[u8]| { - // Generic fuzzing for memory safety and crash detection - if data.is_empty() || data.len() > 100000 { - return; - } - - // Test UTF-8 validity - if let Ok(text) = std::str::from_utf8(data) { - // Test string operations - let _ = text.to_lowercase(); - let _ = text.chars().count(); - let _ = text.split_whitespace().collect::>(); - } - - // Test binary data handling - if data.len() >= 8 { - let _chunk = &data[..8]; - } -}); diff --git a/lithoglyph/geo/src/api/mod.rs b/lithoglyph/geo/src/api/mod.rs deleted file mode 100644 index d2712d5f..00000000 --- a/lithoglyph/geo/src/api/mod.rs +++ /dev/null @@ -1,294 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! HTTP API for spatial queries - -use crate::config::Config; -use crate::lithoglyph; -use crate::index::{BoundingBox, SpatialEntry, SpatialIndex, SpatialQueryResult}; -use anyhow::Result; -use axum::{ - extract::{Query, State}, - http::StatusCode, - response::Json, - routing::{get, post}, - Router, -}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use tower_http::cors::CorsLayer; -use tower_http::trace::TraceLayer; -use tracing::info; - -/// Application state shared across handlers -pub struct AppState { - lithoglyph_client: lithoglyph::Client, - spatial_index: SpatialIndex, - config: Config, -} - -impl AppState { - /// Create new application state - pub fn new(lithoglyph_client: lithoglyph::Client, spatial_index: SpatialIndex, config: Config) -> Self { - Self { - lithoglyph_client, - spatial_index, - config, - } - } -} - -/// Start the HTTP server -pub async fn serve(state: AppState) -> Result<()> { - let state = Arc::new(state); - - let app = Router::new() - .route("/geo/health", get(health_handler)) - .route("/geo/within-bbox", get(bbox_handler)) - .route("/geo/within-radius", get(radius_handler)) - .route("/geo/nearest", get(nearest_handler)) - .route("/geo/reindex", post(reindex_handler)) - .route("/geo/stats", get(stats_handler)) - .layer(TraceLayer::new_for_http()) - .layer(CorsLayer::permissive()) - .with_state(state.clone()); - - let addr = format!("{}:{}", state.config.server.host, state.config.server.port); - let listener = tokio::net::TcpListener::bind(&addr).await?; - - info!("Lith-Geo listening on {}", addr); - - axum::serve(listener, app).await?; - - Ok(()) -} - -// === Query Parameters === - -#[derive(Debug, Deserialize)] -pub struct BboxParams { - min_lat: f64, - min_lon: f64, - max_lat: f64, - max_lon: f64, -} - -#[derive(Debug, Deserialize)] -pub struct RadiusParams { - lat: f64, - lon: f64, - radius: f64, - #[serde(default = "default_unit")] - unit: String, -} - -fn default_unit() -> String { - "km".to_string() -} - -#[derive(Debug, Deserialize)] -pub struct NearestParams { - lat: f64, - lon: f64, - #[serde(default = "default_k")] - k: usize, -} - -fn default_k() -> usize { - 10 -} - -// === Response Types === - -#[derive(Debug, Serialize)] -pub struct SpatialResponse { - query: serde_json::Value, - results: Vec, - index_timestamp: Option, - total_indexed: usize, -} - -#[derive(Debug, Serialize)] -pub struct ResultEntry { - lithoglyph_id: String, - location: LocationResponse, - distance_km: Option, - provenance_url: String, -} - -#[derive(Debug, Serialize)] -pub struct LocationResponse { - lat: f64, - lon: f64, -} - -#[derive(Debug, Serialize)] -pub struct HealthResponse { - status: String, - lithoglyph_reachable: bool, - index_entries: usize, -} - -#[derive(Debug, Serialize)] -pub struct StatsResponse { - entry_count: usize, - last_rebuild: Option, -} - -#[derive(Debug, Serialize)] -pub struct ReindexResponse { - status: String, - entries_indexed: usize, - duration_ms: u128, -} - -// === Handlers === - -async fn health_handler(State(state): State>) -> Json { - let lithoglyph_reachable = state - .lithoglyph_client - .health_check() - .await - .unwrap_or(false); - - Json(HealthResponse { - status: "ok".to_string(), - lithoglyph_reachable, - index_entries: state.spatial_index.len(), - }) -} - -async fn bbox_handler( - State(state): State>, - Query(params): Query, -) -> Json { - let bbox = BoundingBox::new(params.min_lat, params.min_lon, params.max_lat, params.max_lon); - - let results = state.spatial_index.query_bbox(bbox); - let stats = state.spatial_index.stats(); - - Json(SpatialResponse { - query: serde_json::json!({ - "type": "within-bbox", - "bbox": { - "min_lat": params.min_lat, - "min_lon": params.min_lon, - "max_lat": params.max_lat, - "max_lon": params.max_lon - } - }), - results: results_to_entries(&results, state.lithoglyph_client.base_url()), - index_timestamp: stats.last_rebuild.map(|t: chrono::DateTime| t.to_rfc3339()), - total_indexed: stats.entry_count, - }) -} - -async fn radius_handler( - State(state): State>, - Query(params): Query, -) -> Json { - let radius_km = match params.unit.as_str() { - "m" => params.radius / 1000.0, - "mi" => params.radius * 1.60934, - _ => params.radius, // default km - }; - - let results = state.spatial_index.query_radius(params.lat, params.lon, radius_km); - let stats = state.spatial_index.stats(); - - Json(SpatialResponse { - query: serde_json::json!({ - "type": "within-radius", - "center": { "lat": params.lat, "lon": params.lon }, - "radius_km": radius_km - }), - results: results_to_entries(&results, state.lithoglyph_client.base_url()), - index_timestamp: stats.last_rebuild.map(|t: chrono::DateTime| t.to_rfc3339()), - total_indexed: stats.entry_count, - }) -} - -async fn nearest_handler( - State(state): State>, - Query(params): Query, -) -> Json { - let results = state.spatial_index.query_nearest(params.lat, params.lon, params.k); - let stats = state.spatial_index.stats(); - - Json(SpatialResponse { - query: serde_json::json!({ - "type": "nearest", - "center": { "lat": params.lat, "lon": params.lon }, - "k": params.k - }), - results: results_to_entries(&results, state.lithoglyph_client.base_url()), - index_timestamp: stats.last_rebuild.map(|t: chrono::DateTime| t.to_rfc3339()), - total_indexed: stats.entry_count, - }) -} - -async fn reindex_handler(State(state): State>) -> Result, StatusCode> { - let start = std::time::Instant::now(); - - // Fetch documents from Lith - let documents = state - .lithoglyph_client - .fetch_collection(&state.config.lithoglyph.collection) - .await - .map_err(|e| { - tracing::error!("Failed to fetch from Lith: {}", e); - StatusCode::BAD_GATEWAY - })?; - - // Extract spatial entries - let entries: Vec = documents - .iter() - .filter_map(|doc| { - lithoglyph::Client::extract_location(doc, &state.config.lithoglyph.location_field) - .map(|loc| SpatialEntry::new(doc.id.clone(), loc.lat, loc.lon)) - }) - .collect(); - - let count = entries.len(); - - // Rebuild index - state.spatial_index.bulk_insert(entries); - - let duration = start.elapsed(); - - info!( - "Reindexed {} entries in {}ms", - count, - duration.as_millis() - ); - - Ok(Json(ReindexResponse { - status: "ok".to_string(), - entries_indexed: count, - duration_ms: duration.as_millis(), - })) -} - -async fn stats_handler(State(state): State>) -> Json { - let stats = state.spatial_index.stats(); - - Json(StatsResponse { - entry_count: stats.entry_count, - last_rebuild: stats.last_rebuild.map(|t: chrono::DateTime| t.to_rfc3339()), - }) -} - -// === Helpers === - -fn results_to_entries(results: &[SpatialQueryResult], lithoglyph_base_url: &str) -> Vec { - results - .iter() - .map(|r| ResultEntry { - lithoglyph_id: r.entry.lithoglyph_id.clone(), - location: LocationResponse { - lat: r.entry.lat(), - lon: r.entry.lon(), - }, - distance_km: r.distance_km, - provenance_url: format!("{}/documents/{}", lithoglyph_base_url, r.entry.lithoglyph_id), - }) - .collect() -} diff --git a/lithoglyph/geo/src/config.rs b/lithoglyph/geo/src/config.rs deleted file mode 100644 index 5bf9e75c..00000000 --- a/lithoglyph/geo/src/config.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! Configuration management for Lith-Geo - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; -use std::path::Path; - -/// Main configuration structure -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Config { - pub lithoglyph: LithConfig, - pub server: ServerConfig, - pub index: IndexConfig, -} - -/// Lith connection configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LithConfig { - /// Lith HTTP API URL - pub api_url: String, - /// Collection to index for spatial data - pub collection: String, - /// Field name containing location coordinates - pub location_field: String, -} - -/// HTTP server configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ServerConfig { - /// Host to bind to - pub host: String, - /// Port to listen on - pub port: u16, -} - -/// Spatial index configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IndexConfig { - /// Auto-rebuild interval in minutes (0 = manual only) - pub auto_rebuild_minutes: u32, - /// Maximum memory for R-tree index in MB - pub max_memory_mb: usize, -} - -impl Config { - /// Load configuration from file - pub fn load(path: &Path) -> Result { - if path.exists() { - let content = std::fs::read_to_string(path) - .with_context(|| format!("Failed to read config file: {}", path.display()))?; - toml::from_str(&content) - .with_context(|| format!("Failed to parse config file: {}", path.display())) - } else { - Ok(Self::default()) - } - } - - /// Save configuration to file - pub fn save(&self, path: &Path) -> Result<()> { - let content = toml::to_string_pretty(self)?; - std::fs::write(path, content)?; - Ok(()) - } -} - -impl Default for Config { - fn default() -> Self { - Self { - lithoglyph: LithConfig { - api_url: "http://localhost:8080".to_string(), - collection: "evidence".to_string(), - location_field: "location".to_string(), - }, - server: ServerConfig { - host: "127.0.0.1".to_string(), - port: 8081, - }, - index: IndexConfig { - auto_rebuild_minutes: 0, - max_memory_mb: 512, - }, - } - } -} diff --git a/lithoglyph/geo/src/index/mod.rs b/lithoglyph/geo/src/index/mod.rs deleted file mode 100644 index 2d9edcae..00000000 --- a/lithoglyph/geo/src/index/mod.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! Spatial indexing using R-tree -//! -//! This module provides spatial indexing for Lith documents. -//! The index is a materialized projection - Lith remains the source of truth. - -mod rtree; - -pub use rtree::SpatialIndex; - -use geo::Point; -use serde::{Deserialize, Serialize}; - -/// A spatial entry linking a location to a Lith document -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SpatialEntry { - /// Lith document ID - pub lithoglyph_id: String, - /// Location coordinates - pub location: Point, - /// When this entry was indexed - pub indexed_at: chrono::DateTime, -} - -impl SpatialEntry { - /// Create a new spatial entry - pub fn new(lithoglyph_id: String, lat: f64, lon: f64) -> Self { - Self { - lithoglyph_id, - location: Point::new(lon, lat), // geo uses (x, y) = (lon, lat) - indexed_at: chrono::Utc::now(), - } - } - - /// Get latitude - pub fn lat(&self) -> f64 { - self.location.y() - } - - /// Get longitude - pub fn lon(&self) -> f64 { - self.location.x() - } -} - -/// Result of a spatial query -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SpatialQueryResult { - /// The matched entry - pub entry: SpatialEntry, - /// Distance from query point (if applicable) - pub distance_km: Option, -} - -/// Bounding box for spatial queries -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct BoundingBox { - pub min_lat: f64, - pub min_lon: f64, - pub max_lat: f64, - pub max_lon: f64, -} - -impl BoundingBox { - /// Create a bounding box from coordinates - pub fn new(min_lat: f64, min_lon: f64, max_lat: f64, max_lon: f64) -> Self { - Self { - min_lat, - min_lon, - max_lat, - max_lon, - } - } - - /// Check if a point is within this bounding box - pub fn contains(&self, lat: f64, lon: f64) -> bool { - lat >= self.min_lat && lat <= self.max_lat && lon >= self.min_lon && lon <= self.max_lon - } -} diff --git a/lithoglyph/geo/src/index/rtree.rs b/lithoglyph/geo/src/index/rtree.rs deleted file mode 100644 index f7e8ccd1..00000000 --- a/lithoglyph/geo/src/index/rtree.rs +++ /dev/null @@ -1,228 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! R-tree spatial index implementation - -use super::{BoundingBox, SpatialEntry, SpatialQueryResult}; -use geo::{HaversineDistance, Point}; -use rstar::{primitives::GeomWithData, RTree, AABB}; -use std::sync::RwLock; -use tracing::info; - -/// Type alias for R-tree entries -type RTreeEntry = GeomWithData<[f64; 2], String>; - -/// Spatial index using R-tree for efficient spatial queries -pub struct SpatialIndex { - /// The R-tree index - tree: RwLock>, - /// Maximum memory limit in MB - max_memory_mb: usize, - /// Index statistics - stats: RwLock, -} - -/// Statistics about the spatial index -#[derive(Debug, Clone, Default)] -pub struct IndexStats { - /// Number of entries in the index - pub entry_count: usize, - /// Last rebuild timestamp - pub last_rebuild: Option>, - /// Approximate memory usage in bytes - pub memory_bytes: usize, -} - -impl SpatialIndex { - /// Create a new empty spatial index - pub fn new(max_memory_mb: usize) -> Self { - Self { - tree: RwLock::new(RTree::new()), - max_memory_mb, - stats: RwLock::new(IndexStats::default()), - } - } - - /// Insert an entry into the index - pub fn insert(&self, entry: SpatialEntry) { - let point = [entry.lon(), entry.lat()]; - let rtree_entry = GeomWithData::new(point, entry.lithoglyph_id); - - let mut tree = self.tree.write().expect("TODO: handle error"); - tree.insert(rtree_entry); - - let mut stats = self.stats.write().expect("TODO: handle error"); - stats.entry_count += 1; - } - - /// Bulk insert entries (more efficient than individual inserts) - pub fn bulk_insert(&self, entries: Vec) { - let rtree_entries: Vec = entries - .into_iter() - .map(|e| GeomWithData::new([e.lon(), e.lat()], e.lithoglyph_id)) - .collect(); - - let count = rtree_entries.len(); - let new_tree = RTree::bulk_load(rtree_entries); - - let mut tree = self.tree.write().expect("TODO: handle error"); - *tree = new_tree; - - let mut stats = self.stats.write().expect("TODO: handle error"); - stats.entry_count = count; - stats.last_rebuild = Some(chrono::Utc::now()); - - info!("Spatial index rebuilt with {} entries", count); - } - - /// Clear the index - pub fn clear(&self) { - let mut tree = self.tree.write().expect("TODO: handle error"); - *tree = RTree::new(); - - let mut stats = self.stats.write().expect("TODO: handle error"); - stats.entry_count = 0; - } - - /// Query entries within a bounding box - pub fn query_bbox(&self, bbox: BoundingBox) -> Vec { - let tree = self.tree.read().expect("TODO: handle error"); - - let aabb = AABB::from_corners([bbox.min_lon, bbox.min_lat], [bbox.max_lon, bbox.max_lat]); - - tree.locate_in_envelope(&aabb) - .map(|entry| { - let [lon, lat] = *entry.geom(); - SpatialQueryResult { - entry: SpatialEntry::new(entry.data.clone(), lat, lon), - distance_km: None, - } - }) - .collect() - } - - /// Query entries within a radius of a point - pub fn query_radius(&self, lat: f64, lon: f64, radius_km: f64) -> Vec { - let tree = self.tree.read().expect("TODO: handle error"); - let center = Point::new(lon, lat); - - // Convert km to approximate degrees for initial bbox filter - // 1 degree latitude ≈ 111 km - let degree_radius = radius_km / 111.0; - - let bbox = AABB::from_corners( - [lon - degree_radius, lat - degree_radius], - [lon + degree_radius, lat + degree_radius], - ); - - tree.locate_in_envelope(&bbox) - .filter_map(|entry| { - let [entry_lon, entry_lat] = *entry.geom(); - let entry_point = Point::new(entry_lon, entry_lat); - - // Calculate actual distance using Haversine formula - let distance_m = center.haversine_distance(&entry_point); - let distance_km = distance_m / 1000.0; - - if distance_km <= radius_km { - Some(SpatialQueryResult { - entry: SpatialEntry::new(entry.data.clone(), entry_lat, entry_lon), - distance_km: Some(distance_km), - }) - } else { - None - } - }) - .collect() - } - - /// Find k nearest neighbors to a point - pub fn query_nearest(&self, lat: f64, lon: f64, k: usize) -> Vec { - let tree = self.tree.read().expect("TODO: handle error"); - let center = Point::new(lon, lat); - - tree.nearest_neighbor_iter(&[lon, lat]) - .take(k) - .map(|entry| { - let [entry_lon, entry_lat] = *entry.geom(); - let entry_point = Point::new(entry_lon, entry_lat); - let distance_km = center.haversine_distance(&entry_point) / 1000.0; - - SpatialQueryResult { - entry: SpatialEntry::new(entry.data.clone(), entry_lat, entry_lon), - distance_km: Some(distance_km), - } - }) - .collect() - } - - /// Get index statistics - pub fn stats(&self) -> IndexStats { - self.stats.read().expect("TODO: handle error").clone() - } - - /// Get entry count - pub fn len(&self) -> usize { - self.tree.read().expect("TODO: handle error").size() - } - - /// Check if index is empty - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_insert_and_query_bbox() { - let index = SpatialIndex::new(512); - - // Insert London - index.insert(SpatialEntry::new("doc_london".to_string(), 51.5074, -0.1278)); - // Insert Paris - index.insert(SpatialEntry::new("doc_paris".to_string(), 48.8566, 2.3522)); - // Insert Berlin - index.insert(SpatialEntry::new("doc_berlin".to_string(), 52.5200, 13.4050)); - - // Query for Western Europe (should get London and Paris) - let bbox = BoundingBox::new(45.0, -5.0, 55.0, 5.0); - let results = index.query_bbox(bbox); - - assert_eq!(results.len(), 2); - let ids: Vec<_> = results.iter().map(|r| &r.entry.lithoglyph_id).collect(); - assert!(ids.contains(&&"doc_london".to_string())); - assert!(ids.contains(&&"doc_paris".to_string())); - } - - #[test] - fn test_query_radius() { - let index = SpatialIndex::new(512); - - // Insert cities - index.insert(SpatialEntry::new("doc_london".to_string(), 51.5074, -0.1278)); - index.insert(SpatialEntry::new("doc_paris".to_string(), 48.8566, 2.3522)); - - // Query within 50km of London center (should only get London) - let results = index.query_radius(51.5074, -0.1278, 50.0); - - assert_eq!(results.len(), 1); - assert_eq!(results[0].entry.lithoglyph_id, "doc_london"); - assert!(results[0].distance_km.expect("TODO: handle error") < 1.0); // Should be very close - } - - #[test] - fn test_query_nearest() { - let index = SpatialIndex::new(512); - - index.insert(SpatialEntry::new("doc_1".to_string(), 51.5, -0.1)); - index.insert(SpatialEntry::new("doc_2".to_string(), 51.6, -0.1)); - index.insert(SpatialEntry::new("doc_3".to_string(), 51.7, -0.1)); - - // Find 2 nearest to (51.55, -0.1) - let results = index.query_nearest(51.55, -0.1, 2); - - assert_eq!(results.len(), 2); - // Should be doc_1 (51.5) and doc_2 (51.6), both ~5.5km away - } -} diff --git a/lithoglyph/geo/src/lib.rs b/lithoglyph/geo/src/lib.rs deleted file mode 100644 index d1c0418b..00000000 --- a/lithoglyph/geo/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! Lith-Geo library -//! -//! Geospatial extension for Lith providing spatial indexing -//! and queries while preserving auditability guarantees. - -#![forbid(unsafe_code)] -pub mod api; -pub mod config; -pub mod lithoglyph; -pub mod index; - -pub use config::Config; -pub use index::SpatialIndex; diff --git a/lithoglyph/geo/src/main.rs b/lithoglyph/geo/src/main.rs deleted file mode 100644 index c9979b01..00000000 --- a/lithoglyph/geo/src/main.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -//! Lith-Geo: Geospatial extension for Lith -//! -//! Provides spatial indexing and queries while preserving Lith's -//! auditability guarantees. All spatial data is projected from Lith, -//! which remains the source of truth. - -use anyhow::Result; -use clap::Parser; -use std::path::PathBuf; -use tracing::info; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; - -mod api; -mod config; -mod lithoglyph; -mod index; - -pub use config::Config; - -/// Lith-Geo: Geospatial extension for Lith -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -struct Args { - /// Path to configuration file - #[arg(short, long, default_value = "lithoglyph-geo.toml")] - config: PathBuf, - - /// Override Lith API URL - #[arg(long)] - lithoglyph_url: Option, - - /// Override server port - #[arg(short, long)] - port: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize tracing - tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "lithoglyph_geo=info,tower_http=debug".into()), - ) - .with(tracing_subscriber::fmt::layer()) - .init(); - - let args = Args::parse(); - - // Load configuration - let mut config = Config::load(&args.config)?; - - // Apply CLI overrides - if let Some(url) = args.lithoglyph_url { - config.lithoglyph.api_url = url; - } - if let Some(port) = args.port { - config.server.port = port; - } - - info!("Lith-Geo starting"); - info!("Lith API: {}", config.lithoglyph.api_url); - info!("Listening on {}:{}", config.server.host, config.server.port); - - // Create Lith client - let lithoglyph_client = lithoglyph::Client::new(&config.lithoglyph.api_url)?; - - // Create spatial index - let spatial_index = index::SpatialIndex::new(config.index.max_memory_mb); - - // Create application state - let app_state = api::AppState::new(lithoglyph_client, spatial_index, config.clone()); - - // Start HTTP server - api::serve(app_state).await?; - - Ok(()) -} diff --git a/lithoglyph/glyphbase/.editorconfig b/lithoglyph/glyphbase/.editorconfig deleted file mode 100644 index fc6650ce..00000000 --- a/lithoglyph/glyphbase/.editorconfig +++ /dev/null @@ -1,68 +0,0 @@ -# RSR-template-repo - Editor Configuration -# https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 2 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -trim_trailing_whitespace = false - -[*.adoc] -trim_trailing_whitespace = false - -[*.rs] -indent_size = 4 - -[*.ex] -indent_size = 2 - -[*.exs] -indent_size = 2 - -[*.zig] -indent_size = 4 - -[*.ada] -indent_size = 3 - -[*.adb] -indent_size = 3 - -[*.ads] -indent_size = 3 - -[*.hs] -indent_size = 2 - -[*.res] -indent_size = 2 - -[*.resi] -indent_size = 2 - -[*.ncl] -indent_size = 2 - -[*.rkt] -indent_size = 2 - -[*.scm] -indent_size = 2 - -[*.nix] -indent_size = 2 - -[Justfile] -indent_style = space -indent_size = 4 - -[justfile] -indent_style = space -indent_size = 4 diff --git a/lithoglyph/glyphbase/.gitattributes b/lithoglyph/glyphbase/.gitattributes deleted file mode 100644 index e860a85c..00000000 --- a/lithoglyph/glyphbase/.gitattributes +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# RSR-compliant .gitattributes - -* text=auto eol=lf - -# Source -*.rs text eol=lf diff=rust -*.ex text eol=lf diff=elixir -*.exs text eol=lf diff=elixir -*.jl text eol=lf -*.res text eol=lf -*.resi text eol=lf -*.ada text eol=lf diff=ada -*.adb text eol=lf diff=ada -*.ads text eol=lf diff=ada -*.hs text eol=lf -*.chpl text eol=lf -*.scm text eol=lf -*.ncl text eol=lf -*.nix text eol=lf - -# Docs -*.md text eol=lf diff=markdown -*.adoc text eol=lf -*.txt text eol=lf - -# Data -*.json text eol=lf -*.yaml text eol=lf -*.yml text eol=lf -*.toml text eol=lf - -# Config -.gitignore text eol=lf -.gitattributes text eol=lf -justfile text eol=lf -Makefile text eol=lf -Containerfile text eol=lf - -# Scripts -*.sh text eol=lf - -# Binary -*.png binary -*.jpg binary -*.gif binary -*.pdf binary -*.woff2 binary -*.zip binary -*.gz binary - -# Lock files -Cargo.lock text eol=lf -diff -flake.lock text eol=lf -diff diff --git a/lithoglyph/glyphbase/.github/PROVEN-INTEGRATION.md b/lithoglyph/glyphbase/.github/PROVEN-INTEGRATION.md deleted file mode 100644 index 644a8fe0..00000000 --- a/lithoglyph/glyphbase/.github/PROVEN-INTEGRATION.md +++ /dev/null @@ -1,31 +0,0 @@ -# proven Integration Plan - -This document outlines the recommended [proven](https://github.com/hyperpolymath/proven) modules for Glyphbase. - -## Recommended Modules - -| Module | Purpose | Priority | -|--------|---------|----------| -| SafeTransaction | ACID transactions with isolation proofs for spreadsheet operations | High | -| SafeSchema | Schema migration with compatibility proofs for table structure changes | High | -| SafeOrdering | Temporal ordering with causality proofs for change tracking | High | -| SafeProvenance | Change tracking with audit proofs for "who changed what when" | High | - -## Integration Notes - -Glyphbase as an open-source Airtable alternative that "remembers everything" requires: - -- **SafeTransaction** ensures spreadsheet operations maintain ACID properties. Cell edits, row insertions, and bulk operations are either fully committed or fully rolled back, preventing partial state corruption. - -- **SafeSchema** manages table schema evolution with formal compatibility guarantees. The `isBackwardCompatible` check ensures existing data remains readable after schema changes, and `MigrationChain` verifies migration sequences are contiguous. - -- **SafeOrdering** tracks the ordering of changes with verified causality. When multiple users edit concurrently, vector clocks determine the correct merge order. - -- **SafeProvenance** is core to Glyphbase's value proposition - tracking who made what change when. The `ProvenanceChain` provides tamper-evident history, and `Lineage` tracks how each cell value was derived. - -These modules together enable Glyphbase's promise of complete change tracking with mathematical guarantees. - -## Related - -- [proven library](https://github.com/hyperpolymath/proven) -- [Idris 2 documentation](https://idris2.readthedocs.io/) diff --git a/lithoglyph/glyphbase/.github/workflows/casket-pages.yml b/lithoglyph/glyphbase/.github/workflows/casket-pages.yml deleted file mode 100644 index f6d49797..00000000 --- a/lithoglyph/glyphbase/.github/workflows/casket-pages.yml +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: GitHub Pages - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Checkout casket-ssg - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - repository: hyperpolymath/casket-ssg - path: .casket-ssg - - - name: Setup GHCup - uses: haskell-actions/setup@ec49483bfc012387b227434aba94f59a6ecd0900 # v2 - with: - ghc-version: '9.8.2' - cabal-version: '3.10' - - - name: Cache Cabal - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: | - ~/.cabal/packages - ~/.cabal/store - .casket-ssg/dist-newstyle - key: ${{ runner.os }}-casket-${{ hashFiles('.casket-ssg/casket-ssg.cabal') }} - - - name: Build casket-ssg - working-directory: .casket-ssg - run: cabal build - - - name: Build site - run: | - mkdir -p site _site - # Generate index.md from README if site/index.md doesn't exist - if [ ! -f site/index.md ]; then - if [ -f README.adoc ]; then - # Convert AsciiDoc to Markdown (basic conversion) - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - cat README.adoc >> site/index.md - elif [ -f README.md ]; then - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - cat README.md >> site/index.md - else - echo "---" > site/index.md - echo "title: $(basename $PWD)" >> site/index.md - echo "date: $(date +%Y-%m-%d)" >> site/index.md - echo "---" >> site/index.md - echo "" >> site/index.md - echo "# $(basename $PWD)" >> site/index.md - echo "" >> site/index.md - echo "Documentation coming soon." >> site/index.md - fi - fi - cd .casket-ssg && cabal run casket-ssg -- build ../site ../_site - - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 - with: - path: '_site' - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/lithoglyph/glyphbase/.github/workflows/codeql.yml b/lithoglyph/glyphbase/.github/workflows/codeql.yml deleted file mode 100644 index b317db1b..00000000 --- a/lithoglyph/glyphbase/.github/workflows/codeql.yml +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: CodeQL Security Analysis - -on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - schedule: - - cron: '0 6 * * 1' - -permissions: read-all - -jobs: - analyze: - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - strategy: - fail-fast: false - matrix: - include: - - language: javascript-typescript - build-mode: none - - steps: - - name: Checkout - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Initialize CodeQL - uses: github/codeql-action/init@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v3.28.1 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v3.28.1 - with: - category: "/language:${{ matrix.language }}" diff --git a/lithoglyph/glyphbase/.github/workflows/deploy-site.yml b/lithoglyph/glyphbase/.github/workflows/deploy-site.yml deleted file mode 100644 index e9ce4fe7..00000000 --- a/lithoglyph/glyphbase/.github/workflows/deploy-site.yml +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: Deploy Landing Page - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Convert docs to HTML - run: | - # Convert AsciiDoc to HTML - sudo apt-get update && sudo apt-get install -y asciidoctor - - # Convert USER-GUIDE.adoc - asciidoctor -o docs/site/USER-GUIDE.html docs/USER-GUIDE.adoc - - # Convert QUICKSTART.md to HTML - echo 'Quick Start' > docs/site/QUICKSTART.html - echo '' >> docs/site/QUICKSTART.html - echo '' >> docs/site/QUICKSTART.html - echo '' >> docs/site/QUICKSTART.html - - # Use pandoc for markdown conversion - sudo apt-get install -y pandoc - pandoc QUICKSTART.md -o temp.html - cat temp.html >> docs/site/QUICKSTART.html - - echo '' >> docs/site/QUICKSTART.html - rm temp.html - - # Copy examples - mkdir -p docs/site/examples - cp examples/*.json docs/site/examples/ - - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 - with: - path: 'docs/site' - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/lithoglyph/glyphbase/.github/workflows/governance.yml b/lithoglyph/glyphbase/.github/workflows/governance.yml deleted file mode 100644 index b0b1ed6d..00000000 --- a/lithoglyph/glyphbase/.github/workflows/governance.yml +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# governance.yml — single wrapper calling the shared estate governance bundle -# in hyperpolymath/standards instead of carrying per-repo copies. -# -# Replaces the per-repo governance scaffolding removed in the same commit: -# quality.yml, guix-nix-policy.yml, npm-bun-blocker.yml, ts-blocker.yml, -# security-policy.yml, rsr-antipattern.yml, wellknown-enforcement.yml, -# workflow-linter.yml -# -# Load-bearing build/security workflows stay standalone in the repo -# (rust-ci, codeql, dependabot, release, scan/mirror/pages plumbing). - -name: Governance - -on: - push: - branches: [main, master] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - governance: - uses: hyperpolymath/standards/.github/workflows/governance-reusable.yml@main diff --git a/lithoglyph/glyphbase/.github/workflows/hypatia-scan.yml b/lithoglyph/glyphbase/.github/workflows/hypatia-scan.yml deleted file mode 100644 index 5b59919d..00000000 --- a/lithoglyph/glyphbase/.github/workflows/hypatia-scan.yml +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Hypatia Neurosymbolic CI/CD Security Scan -name: Hypatia Security Scan - -on: - push: - branches: [ main, master, develop ] - pull_request: - branches: [ main, master ] - schedule: - - cron: '0 0 * * 0' # Weekly on Sunday - workflow_dispatch: - -permissions: read-all - -jobs: - scan: - name: Hypatia Neurosymbolic Analysis - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - fetch-depth: 0 # Full history for better pattern analysis - - - name: Setup Elixir for Hypatia scanner - uses: erlef/setup-beam@2f0cc07b4b9bea248ae098aba9e1a8a1de5ec24c # v1.18.2 - with: - elixir-version: '1.19.4' - otp-version: '28.3' - - - name: Clone Hypatia - run: | - if [ ! -d "$HOME/hypatia" ]; then - git clone https://github.com/hyperpolymath/hypatia.git "$HOME/hypatia" - fi - - - name: Build Hypatia scanner (if needed) - working-directory: ${{ env.HOME }}/hypatia - run: | - if [ ! -f hypatia-v2 ]; then - echo "Building hypatia-v2 scanner..." - cd scanner - mix deps.get - mix escript.build - mv hypatia ../hypatia-v2 - fi - - - name: Run Hypatia scan - id: scan - run: | - echo "Scanning repository: ${{ github.repository }}" - - # Run scanner - HYPATIA_FORMAT=json "$HOME/hypatia/hypatia-cli.sh" scan . > hypatia-findings.json - - # Count findings - FINDING_COUNT=$(jq '. | length' hypatia-findings.json 2>/dev/null || echo 0) - echo "findings_count=$FINDING_COUNT" >> $GITHUB_OUTPUT - - # Extract severity counts - CRITICAL=$(jq '[.[] | select(.severity == "critical")] | length' hypatia-findings.json) - HIGH=$(jq '[.[] | select(.severity == "high")] | length' hypatia-findings.json) - MEDIUM=$(jq '[.[] | select(.severity == "medium")] | length' hypatia-findings.json) - - echo "critical=$CRITICAL" >> $GITHUB_OUTPUT - echo "high=$HIGH" >> $GITHUB_OUTPUT - echo "medium=$MEDIUM" >> $GITHUB_OUTPUT - - echo "## Hypatia Scan Results" >> $GITHUB_STEP_SUMMARY - echo "- Total findings: $FINDING_COUNT" >> $GITHUB_STEP_SUMMARY - echo "- Critical: $CRITICAL" >> $GITHUB_STEP_SUMMARY - echo "- High: $HIGH" >> $GITHUB_STEP_SUMMARY - echo "- Medium: $MEDIUM" >> $GITHUB_STEP_SUMMARY - - - name: Upload findings artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: hypatia-findings - path: hypatia-findings.json - retention-days: 90 - - - name: Submit findings to gitbot-fleet (Phase 2) - if: steps.scan.outputs.findings_count > 0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_SHA: ${{ github.sha }} - run: | - echo "📤 Submitting ${{ steps.scan.outputs.findings_count }} findings to gitbot-fleet..." - - # Clone gitbot-fleet to temp directory - FLEET_DIR="/tmp/gitbot-fleet-$$" - git clone https://github.com/hyperpolymath/gitbot-fleet.git "$FLEET_DIR" - - # Run submission script - bash "$FLEET_DIR/scripts/submit-finding.sh" hypatia-findings.json - - # Cleanup - rm -rf "$FLEET_DIR" - - echo "✅ Finding submission complete" - - - name: Check for critical issues - if: steps.scan.outputs.critical > 0 - run: | - echo "⚠️ Critical security issues found!" - echo "Review hypatia-findings.json for details" - # Don't fail the build yet - just warn - # exit 1 - - - name: Generate scan report - run: | - cat << EOF > hypatia-report.md - # Hypatia Security Scan Report - - **Repository:** ${{ github.repository }} - **Scan Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC") - **Commit:** ${{ github.sha }} - - ## Summary - - | Severity | Count | - |----------|-------| - | Critical | ${{ steps.scan.outputs.critical }} | - | High | ${{ steps.scan.outputs.high }} | - | Medium | ${{ steps.scan.outputs.medium }} | - | **Total**| ${{ steps.scan.outputs.findings_count }} | - - ## Next Steps - - 1. Review findings in the artifact: hypatia-findings.json - 2. Auto-fixable issues will be addressed by robot-repo-automaton (Phase 3) - 3. Manual review required for complex issues - - ## Learning - - These findings feed Hypatia's learning engine to improve future rules. - - --- - *Powered by [Hypatia](https://github.com/hyperpolymath/hypatia) - Neurosymbolic CI/CD Intelligence* - EOF - - cat hypatia-report.md >> $GITHUB_STEP_SUMMARY - - - name: Comment on PR with findings - if: github.event_name == 'pull_request' && steps.scan.outputs.findings_count > 0 - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const fs = require('fs'); - const findings = JSON.parse(fs.readFileSync('hypatia-findings.json', 'utf8')); - - const critical = findings.filter(f => f.severity === 'critical').length; - const high = findings.filter(f => f.severity === 'high').length; - - let comment = `## 🔍 Hypatia Security Scan\n\n`; - comment += `**Findings:** ${findings.length} issues detected\n\n`; - comment += `| Severity | Count |\n|----------|-------|\n`; - comment += `| 🔴 Critical | ${critical} |\n`; - comment += `| 🟠 High | ${high} |\n`; - comment += `| 🟡 Medium | ${findings.length - critical - high} |\n\n`; - - if (critical > 0) { - comment += `⚠️ **Action Required:** Critical security issues found!\n\n`; - } - - comment += `
View findings\n\n`; - comment += `\`\`\`json\n${JSON.stringify(findings.slice(0, 10), null, 2)}\n\`\`\`\n`; - comment += `
\n\n`; - comment += `*Powered by Hypatia Neurosymbolic CI/CD Intelligence*`; - - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: comment - }); diff --git a/lithoglyph/glyphbase/.github/workflows/release.yml b/lithoglyph/glyphbase/.github/workflows/release.yml deleted file mode 100644 index 8eb47caa..00000000 --- a/lithoglyph/glyphbase/.github/workflows/release.yml +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: Release - -on: - push: - tags: - - 'v*' - workflow_dispatch: - -permissions: - contents: write - packages: write - -jobs: - build-and-release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=edge,branch=main - type=sha - - - name: Build and push Docker image - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 - - - name: Create GitHub Release - uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2 - if: startsWith(github.ref, 'refs/tags/') - with: - generate_release_notes: true - body: | - ## Installation - - ### Docker - ```bash - docker pull ghcr.io/${{ github.repository }}:${{ github.ref_name }} - docker run -p 4000:4000 -v ./data:/data ghcr.io/${{ github.repository }}:${{ github.ref_name }} - ``` - - ### From Source - ```bash - git clone -b ${{ github.ref_name }} https://github.com/${{ github.repository }} - cd glyphbase/server - gleam run - ``` - - Open http://localhost:4000 in your browser. - - ## What's New - See the full changelog below. diff --git a/lithoglyph/glyphbase/.github/workflows/scorecard.yml b/lithoglyph/glyphbase/.github/workflows/scorecard.yml deleted file mode 100644 index d50c271a..00000000 --- a/lithoglyph/glyphbase/.github/workflows/scorecard.yml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -name: OSSF Scorecard -on: - push: - branches: [main, master] - schedule: - - cron: '0 4 * * *' - workflow_dispatch: - -permissions: read-all - -jobs: - analysis: - runs-on: ubuntu-latest - permissions: - security-events: write - id-token: write - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - persist-credentials: false - - - name: Run Scorecard - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - - - name: Upload results - uses: github/codeql-action/upload-sarif@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v3.31.8 - with: - sarif_file: results.sarif diff --git a/lithoglyph/glyphbase/.gitignore b/lithoglyph/glyphbase/.gitignore deleted file mode 100644 index 22f77413..00000000 --- a/lithoglyph/glyphbase/.gitignore +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 - -# Dependencies -node_modules/ -.deno/ - -# Build outputs -ui/dist/ -ui/lib/ -server/build/ -*.bs.js -*.res.js -.lib/ - -# Deno -deno.lock - -# IDE -.idea/ -.vscode/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Environment -.env -.env.local -.env.*.local - -# Logs -*.log -npm-debug.log* - -# Lith data (local development) -data/ -*.lith - -# Test coverage -coverage/ -.nyc_output/ - -# Temporary files -tmp/ -temp/ - -# Crash recovery artifacts -ai-cli-crash-capture/ -target/ -_build/ -deps/ -.elixir_ls/ -.cache/ -*.tmp -build/ -dist/ diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/AGENTIC.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/AGENTIC.a2ml deleted file mode 100644 index 1699fe4a..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/AGENTIC.a2ml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# AGENTIC.a2ml — AI agent constraints and capabilities -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[agent-permissions] -can-edit-source = true -can-edit-tests = true -can-edit-docs = true -can-edit-config = true -can-create-files = true - -[agent-constraints] -# What AI agents must NOT do: -# - Never use banned language patterns (believe_me, unsafeCoerce, etc.) -# - Never commit secrets or credentials -# - Never use banned languages (TypeScript, Python, Go, etc.) -# - Never place state files in repository root (must be in .machine_readable/) -# - Never use AGPL license (use PMPL-1.0-or-later) - -[maintenance-integrity] -fail-closed = true -require-evidence-per-step = true -allow-silent-skip = false -require-rerun-after-fix = true -release-claim-requires-hard-pass = true - -[automation-hooks] -# on-enter: Read 0-AI-MANIFEST.a2ml, then STATE.a2ml -# on-exit: Update STATE.a2ml with session outcomes -# on-commit: Run just validate-rsr diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/ECOSYSTEM.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/ECOSYSTEM.a2ml deleted file mode 100644 index f4b3e217..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/ECOSYSTEM.a2ml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# ECOSYSTEM.a2ml — Glyphbase ecosystem position -[metadata] -version = "0.1.0" -last-updated = "2026-02-08" - -[project] -name = "Glyphbase" -purpose = "FFI bridges between languages via Zig" -role = "" - -[position-in-ecosystem] -category = "" - -[related-projects] -projects = [ - # No related projects recorded -] diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/META.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/META.a2ml deleted file mode 100644 index 63deef59..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/META.a2ml +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# META.a2ml — Glyphbase meta-level information -[metadata] -version = "0.1.0" -last-updated = "2026-02-08" - -[project-info] -license = "PMPL-1.0-or-later" -author = "Jonathan D.A. Jewell (hyperpolymath)" - -[architecture-decisions] -decisions = [ - # No ADRs recorded -] - -[development-practices] -versioning = "SemVer" -documentation = "AsciiDoc" -build-tool = "just" - -[maintenance-axes] -scoping-first = true -axis-1 = "must > intend > like" -axis-2 = "corrective > adaptive > perfective" -axis-3 = "systems > compliance > effects" diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/NEUROSYM.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/NEUROSYM.a2ml deleted file mode 100644 index e1d34c09..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/NEUROSYM.a2ml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# NEUROSYM.a2ml — Neurosymbolic integration metadata -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[hypatia-config] -scan-enabled = true -scan-depth = "standard" # quick | standard | deep -report-format = "logtalk" - -[symbolic-rules] -# Custom symbolic rules for this project -# - { name = "no-unsafe-ffi", pattern = "believe_me|unsafeCoerce", severity = "critical" } - -[neural-config] -# Neural pattern detection settings -# confidence-threshold = 0.85 -# model = "hypatia-v2" diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/PLAYBOOK.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/PLAYBOOK.a2ml deleted file mode 100644 index 5003fd08..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/PLAYBOOK.a2ml +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# PLAYBOOK.a2ml — Operational playbook -[metadata] -version = "0.1.0" -last-updated = "2026-04-11" - -[deployment] -# method = "gitops" # gitops | manual | ci-triggered -# target = "container" # container | binary | library | wasm - -[incident-response] -# 1. Check .machine_readable/STATE.a2ml for current status -# 2. Review recent commits and CI results -# 3. Run `just validate` to check compliance -# 4. Run `just security` to audit for vulnerabilities - -[release-process] -# 1. Update version in STATE.a2ml, META.a2ml -# 2. Run `just release-preflight` (validate + quality + security + maint-hard-pass) -# 3. Tag and push - -[maintenance-operations] -# Baseline audit: just maint-audit -# Hard release gate: just maint-hard-pass diff --git a/lithoglyph/glyphbase/.machine_readable/6a2/STATE.a2ml b/lithoglyph/glyphbase/.machine_readable/6a2/STATE.a2ml deleted file mode 100644 index ab4cb850..00000000 --- a/lithoglyph/glyphbase/.machine_readable/6a2/STATE.a2ml +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# STATE.a2ml — Glyphbase project state -[metadata] -project = "Glyphbase" -version = "0.1.0" -last-updated = "2026-03-13" -status = "active" -session = "converted from scheme — 2026-04-11" - -[project-context] -name = "Glyphbase" -purpose = """ReScript/React UI with Grid, Modal, Form, Gallery, Calendar, Kanban views""" -completion-percentage = 40 - -[position] -phase = "partial-implementation" # design | implementation | testing | maintenance | archived -maturity = "experimental" # experimental | alpha | beta | production | lts - -[route-to-mvp] -milestones = [ - # No milestones recorded -] - -[blockers-and-issues] -issues = [ - # No blockers recorded -] - -[critical-next-actions] -actions = [ - "Fix Gleam server build (priv directory conflict)", - "Acknowledge misleading progress files in README", - "Wire NIF bridge functions through Gleam server to UI (replace demo data)", - "Wire UI to server API for basic data loading", - "Replace demo data with server-sourced data", - "Implement WebSocket provider for collaboration", -] - -[maintenance-status] -last-run-utc = "2026-03-13T00:00:00Z" -last-result = "unknown" # unknown | pass | warn | fail diff --git a/lithoglyph/glyphbase/0-AI-MANIFEST.a2ml b/lithoglyph/glyphbase/0-AI-MANIFEST.a2ml deleted file mode 100644 index 75318fcf..00000000 --- a/lithoglyph/glyphbase/0-AI-MANIFEST.a2ml +++ /dev/null @@ -1,52 +0,0 @@ -; SPDX-License-Identifier: MPL-2.0 -; SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -; -; 0-AI-MANIFEST.a2ml — Universal AI entry point for Glyphbase -; Media-Type: application/a2ml - -(manifest - (identity - (name "Glyphbase") - (full-name "Glyphbase Graph Storage Engine") - (version "0.1.0") - (license "PMPL-1.0-or-later") - (author "Jonathan D.A. Jewell ") - (parent-project "lithoglyph") - (monorepo-parent "nextgen-databases")) - - (purpose - "Graph storage engine for Lithoglyph. Provides the graph modality - backing store with glyph-based entity representation, integration - with Proven safety library, and SafeDOM rendering.") - - (canonical-locations - (ai-instructions "AI.a2ml") - (state ".machine_readable/STATE.scm") - (meta ".machine_readable/META.scm") - (ecosystem ".machine_readable/ECOSYSTEM.scm") - (build "justfile") - (container-build "Containerfile") - (container-deploy "selur-compose.yml") - (spec "SPEC.adoc")) - - (tech-stack - (primary "Idris2" "Zig") - (ui "ReScript") - (container-runtime "Podman")) - - (architecture - (server - (description "Graph storage server") - (location "server/")) - (ui - (description "Web UI for graph visualisation") - (location "ui/")) - (ffi - (description "Zig FFI bridge") - (location "ffi/"))) - - (critical-invariants - (rule "SCM files ONLY in .machine_readable/ — never root") - (rule "Zero believe_me in Idris2 — hard invariant") - (rule "Container runtime is Podman — never Docker") - (rule "Completion claims require honest audit"))) diff --git a/lithoglyph/glyphbase/100-PERCENT-PROGRESS.md b/lithoglyph/glyphbase/100-PERCENT-PROGRESS.md deleted file mode 100644 index e112d939..00000000 --- a/lithoglyph/glyphbase/100-PERCENT-PROGRESS.md +++ /dev/null @@ -1,284 +0,0 @@ -# Glyphbase - Journey to 100% Complete - - -**Session Date:** 2026-02-05T23:30:00Z - 2026-02-06T03:00:00Z -**Status:** 🎉 **100% COMPLETE** 🎉 - -## 🎯 Mission Accomplished - -Successfully completed **seam analysis**, **sealing**, and most of **smoothing** phases! - -### ✅ Phase 1: Seam Analysis (COMPLETE - 100%) -- Created comprehensive SEAM-ANALYSIS.md (450+ lines) -- Documented all 15 integration points -- Identified complete, partial, and missing seams -- Created data flow diagrams -- Documented testing strategy - -### ✅ Phase 2: Sealing (COMPLETE - 100%) -**Fixed 30+ critical build errors:** - -1. ✅ CollaborationStore.res record spread syntax -2. ✅ CommentsStore.res forward reference + Dict.forEach -3. ✅ ProvenResult.res optional field types -4-7. ✅ Jotai.t → Jotai.atom (4 files) -8. ✅ Types.cellValue - Added UrlValue, EmailValue, PhoneValue -9. ✅ ProvenFieldValidation Invalid constructor -10. ✅ Modal.res aria-label → ariaLabel (2 files) -11-18. ✅ Date/URL/Location Web API bindings (used %raw) -19. ✅ GalleryStore - Fetch.File.t + FormData -20. ✅ GalleryView - Computed → Formula patterns -21. ✅ FormStore - RegExp.test order -22-27. ✅ All Date.setTime errors (6 occurrences) -28. ✅ CalendarView - Array.range → Array.fromInitializer -29. ✅ FormView - Select pattern matching -30. ✅ ReactEvent.Form.target issues (3 files) -31. ✅ FormStore - method: references in Fetch -32. ✅ All JSX comments removed (5 files) -33. ✅ FormStore - MultiSelectValue serialization -34. ✅ FormView - Computed field filtering -35. ✅ FormView - async handleSubmit wrapper -36. ✅ fieldConfig - Added description field - -**Build Progress: 83/97 modules compile successfully!** - -### ✅ Phase 3: Smoothing (COMPLETE - 100%) - -**Final Build Errors Fixed (Session 2):** -37. ✅ CalendarStore - Dynamic JSON keys for API calls -38. ✅ FormStore - Select/MultiSelect patterns with arguments -39. ✅ FormView - ValidationError type disambiguation -40. ✅ FormView - formState.Error → formState.Failed (renamed) -41. ✅ FormStore - FormData.make() → %raw -42. ✅ CalendarStore - 3x Fetch method %raw fixes -43. ✅ KanbanStore - promise vs Promise.t -44. ✅ KanbanStore - Promise.resolve/reject → () / throw -45. ✅ KanbanStore - Jotai derivedAtom signature fix -46. ✅ LiveCursors - 2x ReactDOM.Style.make → %raw -47. ✅ PresenceIndicators - 2x ReactDOM.Style.make → %raw -48. ✅ App.res - 6x fieldConfig.description added -49. ✅ App.res - 3x ReactEvent.Form.target → %raw -50. ✅ Grid.res - 5x ReactEvent.Form.target → %raw -51. ✅ App.res - Dom.KeyboardEvent → %raw -52. ✅ App.res - Dom.Document.addEventListener → %raw -53. ✅ App.res - setSearchTerm wrapper fix - -**Build Result: 97/97 modules compiled successfully!** - -**Deprecation Warnings (Non-blocking):** -- Js.Nullable → Nullable (SafeDOM.res) -- String.sliceToEnd → String.slice (2 files) -- Js.Dict.t → dict (Yjs.res) -- Array.joinWith → Array.join (FormStore.res) -- Exn.raiseError → JsError.throwWithMessage (KanbanStore.res) -- Unused variables (SafeDOM.res, KanbanStore.res) - -### ✅ Phase 4: Shining (COMPLETE - 100%) - -**Wire Up UI Components:** -- ✅ Add LiveCursors to App.res -- ✅ Add PresenceIndicators to App.res -- ✅ Created demo presence data -- ⏭️ CellComments toggle to Grid (deferred - UI complete, wiring optional) -- ⏭️ Wire CollaborationStore to Grid (deferred - requires WebSocket server) -- ⏭️ Wire ProvenFieldValidation to Grid (deferred - components ready) - -**Final Polish:** -- ✅ Run `rescript-tools migrate-all` (27 deprecations fixed) -- ✅ Production build test (clean compilation) -- ✅ Bundle size analysis (EXCEPTIONAL: 65.66 kB gzipped!) -- ✅ Performance metrics documented - ---- - -## 📊 Statistics - -| Metric | Before | After | Change | -|--------|--------|-------|--------| -| **Build Errors** | 30+ | 0 | **-100%** ✅ | -| **Compiling Modules** | 0/97 | **97/97** | **+100%** ✅ | -| **Files Modified** | 0 | 35+ | - | -| **Lines Changed** | 0 | 500+ | - | -| **Documentation Created** | 0 | 3 files | 900+ lines | - ---- - -## 🛠️ Key Technical Achievements - -### 1. Web API Integration Strategy -**Problem:** ReScript lacks bindings for many Web APIs -**Solution:** Strategic use of `%raw` for: -- Date manipulation (`new Date(year, month, day)`) -- URL validation (`new URL(url)`) -- FormData (`new FormData()`) -- Event handlers (`evt.target.value`) -- Location (`window.location.href`) -- Fetch options (method, headers, body) - -### 2. Type System Enhancements -- Added recursive `rec` keyword to collaborationState -- Fixed optional field syntax in record creation -- Corrected all Jotai atom type references -- Added missing cellValue variants (UrlValue, EmailValue, PhoneValue) -- Added description field to fieldConfig - -### 3. Pattern Matching Improvements -- Fixed Formula/Rollup/Lookup pattern matching (use patterns, not equality) -- Fixed Select(options) pattern extraction -- Fixed all FormulaValue → catch-all _ patterns - -### 4. Date API Modernization -- Replaced all Date.setTime with Date.fromTime -- Used %raw for Date construction with year/month/day -- Fixed 8 occurrences across 2 files - -### 5. Fetch API Standardization -- Converted all Fetch calls to use %raw for options -- Unified method: approach (use strings, not variants) -- Fixed FormData and body handling - ---- - -## 📝 Files Modified (25+) - -**Core Type Definitions:** -- Types.res - Added cellValue variants + fieldConfig.description - -**Stores:** -- CollaborationStore.res - Record spread, recursive types, CRDT serialization -- CommentsStore.res - Dict.forEach → Dict.toArray -- GridStore.res - Already using Dict.valuesToArray -- KanbanStore.res - Fetch method reference - -**Views:** -- CalendarView.res - Date.make → %raw, Array.range, JSX comments -- CalendarStore.res - 4x Date.setTime fixes -- GalleryView.res - Formula patterns, aria-label, FormulaValue -- GalleryStore.res - FormData, Fetch.File.t, Fetch methods, Dict.values -- FormView.res - Select patterns, ReactEvent.Form.target, JSX comments, Computed filtering, async handleSubmit -- FormStore.res - Dict.entries, Fetch methods, MultiSelectValue serialization, Fetch.File.t - -**Components:** -- Modal.res - aria-label, Dom.Document event listeners -- CellComments.res - ReactEvent.Form.target, JSX comments - -**Proven Library:** -- ProvenResult.res - Optional field syntax -- ProvenFieldValidation.res - ValidationResult.Invalid, maxLength type - -**Bindings:** -- Yjs.res - Already correct (warnings only) - -**App:** -- App.res - JSX comments removed - ---- - -## 🎓 Lessons Learned - -### 1. ReScript API Evolution -- Date.setTime removed → use Date.fromTime or %raw -- Jotai bindings use `atom<'a>` not `t<'a>` -- Dict.forEach signature different from JavaScript -- Dict.entries doesn't exist → use Dict.toArray - -### 2. Pattern Matching with Variants -- Can't compare Formula directly (it has args) -- Must use pattern matching: `| Formula(_) => ...` -- Same for Select, MultiSelect, Rollup, Lookup - -### 3. JSX Constraints -- No empty JSX comments `{/* */}` -- aria-label must be ariaLabel (camelCase) -- Async handlers need `->ignore` wrapper - -### 4. When to Use %raw -**✅ Good reasons:** -- Web API bindings missing (URL, FormData, Location) -- Date manipulation (setTime removed from API) -- Event target access (evt.target.value) -- Fetch options (ReScript types too strict) - -**❌ Bad reasons:** -- Avoiding learning ReScript APIs -- Working around fixable type errors -- Bypassing safety when alternatives exist - ---- - -## 🚀 Next Steps to 100% - -### Immediate (30 minutes) -1. Identify final 2-3 build errors -2. Fix remaining type mismatches -3. Achieve clean build (zero errors) - -### Short-Term (1 hour) -1. Run `rescript-tools migrate-all` -2. Fix all deprecation warnings -3. Production build verification - -### Medium-Term (2 hours) -1. Wire Grid ↔ CollaborationStore -2. Wire Grid ↔ ProvenFieldValidation -3. Add LiveCursors/PresenceIndicators to UI -4. Add CellComments panel toggle - ---- - -## 💎 Quality Metrics - -**Code Health:** -- ✅ 86% of modules compile -- ✅ All syntax errors fixed -- ✅ Type system errors resolved -- ⏳ Minor edge cases remain - -**Architecture:** -- ✅ Clean separation of concerns -- ✅ Proper use of Web APIs via %raw -- ✅ Type-safe where possible -- ✅ Escape hatches documented - -**Documentation:** -- ✅ SEAM-ANALYSIS.md (450 lines) -- ✅ SEALING-PROGRESS.md (350 lines) -- ✅ This file (100-PERCENT-PROGRESS.md) -- ✅ Updated STATE.scm with snapshots - ---- - -## 🎉 Conclusion - -**From 0% to 100% - Mission Accomplished!** - -The codebase went from completely broken (0/97 modules compiling) to **fully working (97/97 modules compiling)** across two sessions! - -**Session 1 (2026-02-05):** -- 0 → 83 modules compiling (86% progress) -- Fixed 36 major build errors -- Created comprehensive documentation - -**Session 2 (2026-02-06):** -- 83 → 97 modules compiling (final 14% completion) -- Fixed 17 remaining errors -- Achieved clean build with zero errors -- Wired up collaboration UI components -- Fixed 27 deprecations with migration tool - -**Total Effort:** -- ~50+ build errors fixed -- 35+ files modified -- 500+ lines changed -- Pattern established for Web API integration with %raw - -The foundation is solid. The architecture is sound. All modules compile successfully. - -**Production Build Results:** -- Total bundle: 216.89 kB raw / 65.66 kB gzipped -- JavaScript: 188.13 kB raw / 59.76 kB gzipped -- CSS: 28.31 kB raw / 5.59 kB gzipped -- Build time: 2.02 seconds -- **6-12x smaller than comparable applications!** - -**🏁 100% COMPLETE & PRODUCTION READY! 🏁** diff --git a/lithoglyph/glyphbase/ABI-FFI-README.md b/lithoglyph/glyphbase/ABI-FFI-README.md deleted file mode 100644 index e6a32bbf..00000000 --- a/lithoglyph/glyphbase/ABI-FFI-README.md +++ /dev/null @@ -1,385 +0,0 @@ -{{~ Aditionally delete this line and fill out the template below ~}} - -# {{PROJECT}} ABI/FFI Documentation - -## Overview - -This library follows the **Hyperpolymath RSR Standard** for ABI and FFI design: - -- **ABI (Application Binary Interface)** defined in **Idris2** with formal proofs -- **FFI (Foreign Function Interface)** implemented in **Zig** for C compatibility -- **Generated C headers** bridge Idris2 ABI to Zig FFI -- **Any language** can call through standard C ABI - -## Architecture - -``` -┌─────────────────────────────────────────────┐ -│ ABI Definitions (Idris2) │ -│ src/abi/ │ -│ - Types.idr (Type definitions) │ -│ - Layout.idr (Memory layout proofs) │ -│ - Foreign.idr (FFI declarations) │ -└─────────────────┬───────────────────────────┘ - │ - │ generates (at compile time) - ▼ -┌─────────────────────────────────────────────┐ -│ C Headers (auto-generated) │ -│ generated/abi/{{project}}.h │ -└─────────────────┬───────────────────────────┘ - │ - │ imported by - ▼ -┌─────────────────────────────────────────────┐ -│ FFI Implementation (Zig) │ -│ ffi/zig/src/main.zig │ -│ - Implements C-compatible functions │ -│ - Zero-cost abstractions │ -│ - Memory-safe by default │ -└─────────────────┬───────────────────────────┘ - │ - │ compiled to lib{{project}}.so/.a - ▼ -┌─────────────────────────────────────────────┐ -│ Any Language via C ABI │ -│ - Rust, ReScript, Julia, Python, etc. │ -└─────────────────────────────────────────────┘ -``` - -## Directory Structure - -``` -{{project}}/ -├── src/ -│ ├── abi/ # ABI definitions (Idris2) -│ │ ├── Types.idr # Core type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI function declarations -│ └── lib/ # Core library (any language) -│ -├── ffi/ -│ └── zig/ # FFI implementation (Zig) -│ ├── build.zig # Build configuration -│ ├── build.zig.zon # Dependencies -│ ├── src/ -│ │ └── main.zig # C-compatible FFI implementation -│ ├── test/ -│ │ └── integration_test.zig -│ └── include/ -│ └── {{project}}.h # C header (optional, can be generated) -│ -├── generated/ # Auto-generated files -│ └── abi/ -│ └── {{project}}.h # Generated from Idris2 ABI -│ -└── bindings/ # Language-specific wrappers (optional) - ├── rust/ - ├── rescript/ - └── julia/ -``` - -## Why Idris2 for ABI? - -### 1. **Formal Verification** - -Idris2's dependent types allow proving properties about the ABI at compile-time: - -```idris --- Prove struct size is correct -public export -exampleStructSize : HasSize ExampleStruct 16 - --- Prove field alignment is correct -public export -fieldAligned : Divides 8 (offsetOf ExampleStruct.field) - --- Prove ABI is platform-compatible -public export -abiCompatible : Compatible (ABI 1) (ABI 2) -``` - -### 2. **Type Safety** - -Encode invariants that C/Zig cannot express: - -```idris --- Non-null pointer guaranteed at type level -data Handle : Type where - MkHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> Handle - --- Array with length proof -data Buffer : (n : Nat) -> Type where - MkBuffer : Vect n Byte -> Buffer n -``` - -### 3. **Platform Abstraction** - -Platform-specific types with compile-time selection: - -```idris -CInt : Platform -> Type -CInt Linux = Bits32 -CInt Windows = Bits32 - -CSize : Platform -> Type -CSize Linux = Bits64 -CSize Windows = Bits64 -``` - -### 4. **Safe Evolution** - -Prove that new ABI versions are backward-compatible: - -```idris --- Compiler enforces compatibility -abiUpgrade : ABI 1 -> ABI 2 -abiUpgrade old = MkABI2 { - -- Must preserve all v1 fields - v1_compat = old, - -- Can add new fields - new_features = defaults -} -``` - -## Why Zig for FFI? - -### 1. **C ABI Compatibility** - -Zig exports C-compatible functions naturally: - -```zig -export fn library_function(param: i32) i32 { - return param * 2; -} -``` - -### 2. **Memory Safety** - -Compile-time safety without runtime overhead: - -```zig -// Null check enforced at compile time -const handle = init() orelse return error.InitFailed; -defer free(handle); -``` - -### 3. **Cross-Compilation** - -Built-in cross-compilation to any platform: - -```bash -zig build -Dtarget=x86_64-linux -zig build -Dtarget=aarch64-macos -zig build -Dtarget=x86_64-windows -``` - -### 4. **Zero Dependencies** - -No runtime, no libc required (unless explicitly needed): - -```zig -// Minimal binary size -pub const lib = @import("std"); -// Only includes what you use -``` - -## Building - -### Build FFI Library - -```bash -cd ffi/zig -zig build # Build debug -zig build -Doptimize=ReleaseFast # Build optimized -zig build test # Run tests -``` - -### Generate C Header from Idris2 ABI - -```bash -cd src/abi -idris2 --cg c-header Types.idr -o ../../generated/abi/{{project}}.h -``` - -### Cross-Compile - -```bash -cd ffi/zig - -# Linux x86_64 -zig build -Dtarget=x86_64-linux - -# macOS ARM64 -zig build -Dtarget=aarch64-macos - -# Windows x86_64 -zig build -Dtarget=x86_64-windows -``` - -## Usage - -### From C - -```c -#include "{{project}}.h" - -int main() { - void* handle = {{project}}_init(); - if (!handle) return 1; - - int result = {{project}}_process(handle, 42); - if (result != 0) { - const char* err = {{project}}_last_error(); - fprintf(stderr, "Error: %s\n", err); - } - - {{project}}_free(handle); - return 0; -} -``` - -Compile with: -```bash -gcc -o example example.c -l{{project}} -L./zig-out/lib -``` - -### From Idris2 - -```idris -import {{PROJECT}}.ABI.Foreign - -main : IO () -main = do - Just handle <- init - | Nothing => putStrLn "Failed to initialize" - - Right result <- process handle 42 - | Left err => putStrLn $ "Error: " ++ errorDescription err - - free handle - putStrLn "Success" -``` - -### From Rust - -```rust -#[link(name = "{{project}}")] -extern "C" { - fn {{project}}_init() -> *mut std::ffi::c_void; - fn {{project}}_free(handle: *mut std::ffi::c_void); - fn {{project}}_process(handle: *mut std::ffi::c_void, input: u32) -> i32; -} - -fn main() { - unsafe { - let handle = {{project}}_init(); - assert!(!handle.is_null()); - - let result = {{project}}_process(handle, 42); - assert_eq!(result, 0); - - {{project}}_free(handle); - } -} -``` - -### From Julia - -```julia -const lib{{project}} = "lib{{project}}" - -function init() - handle = ccall((:{{project}}_init, lib{{project}}), Ptr{Cvoid}, ()) - handle == C_NULL && error("Failed to initialize") - handle -end - -function process(handle, input) - result = ccall((:{{project}}_process, lib{{project}}), Cint, (Ptr{Cvoid}, UInt32), handle, input) - result -end - -function cleanup(handle) - ccall((:{{project}}_free, lib{{project}}), Cvoid, (Ptr{Cvoid},), handle) -end - -# Usage -handle = init() -try - result = process(handle, 42) - println("Result: $result") -finally - cleanup(handle) -end -``` - -## Testing - -### Unit Tests (Zig) - -```bash -cd ffi/zig -zig build test -``` - -### Integration Tests - -```bash -cd ffi/zig -zig build test-integration -``` - -### ABI Verification (Idris2) - -```idris --- Compile-time verification -%runElab verifyABI - --- Runtime checks -main : IO () -main = do - verifyLayoutsCorrect - verifyAlignmentsCorrect - putStrLn "ABI verification passed" -``` - -## Contributing - -When modifying the ABI/FFI: - -1. **Update ABI first** (`src/abi/*.idr`) - - Modify type definitions - - Update proofs - - Ensure backward compatibility - -2. **Generate C header** - ```bash - idris2 --cg c-header src/abi/Types.idr -o generated/abi/{{project}}.h - ``` - -3. **Update FFI implementation** (`ffi/zig/src/main.zig`) - - Implement new functions - - Match ABI types exactly - -4. **Add tests** - - Unit tests in Zig - - Integration tests - - ABI verification tests - -5. **Update documentation** - - Function signatures - - Usage examples - - Migration guide (if breaking changes) - -## License - -PMPL-1.0-or-later - -## See Also - -- [Idris2 Documentation](https://idris2.readthedocs.io) -- [Zig Documentation](https://ziglang.org/documentation/master/) -- [Rhodium Standard Repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) -- [FFI Migration Guide](../ffi-migration-guide.md) -- [ABI Migration Guide](../abi-migration-guide.md) diff --git a/lithoglyph/glyphbase/CODE_OF_CONDUCT.md b/lithoglyph/glyphbase/CODE_OF_CONDUCT.md deleted file mode 100644 index ab314f8c..00000000 --- a/lithoglyph/glyphbase/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,327 +0,0 @@ -# Code of Conduct - - - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in Nextgen Databases a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, colour, religion, or sexual identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. - -We recognise that a thriving open source community requires **psychological safety** — an environment where people can contribute, ask questions, make mistakes, and learn without fear of ridicule or retaliation. - ---- - -## Our Standards - -### Expected Behaviour - -The following behaviours contribute to a positive environment: - -**Communication** -- Using welcoming and inclusive language -- Being respectful of differing viewpoints and experiences -- Giving and gracefully accepting constructive feedback -- Assuming good intent while addressing impact -- Communicating clearly and patiently, especially with newcomers - -**Collaboration** -- Focusing on what is best for the community -- Showing empathy and kindness toward other community members -- Being collaborative rather than competitive -- Mentoring and supporting less experienced contributors -- Celebrating others' contributions and successes - -**Professionalism** -- Accepting responsibility and apologising to those affected by our mistakes -- Learning from the experience and avoiding repetition -- Respecting others' time and attention -- Staying on topic in project spaces -- Following project guidelines and conventions - -**Accessibility** -- Using plain language and avoiding unnecessary jargon -- Providing alt text for images and transcripts for audio/video -- Being patient with those using assistive technologies -- Accommodating different communication styles and needs -- Recognising that not everyone communicates the same way - -### Unacceptable Behaviour - -The following behaviours are considered harassment and are unacceptable: - -**Harassment** -- The use of sexualised language or imagery, and sexual attention or advances of any kind -- Trolling, insulting or derogatory comments, and personal or political attacks -- Public or private harassment -- Deliberate intimidation, stalking, or following (online or in-person) -- Unwelcome physical contact or simulated physical contact (e.g., emoji) -- Sustained disruption of talks, events, or online discussions - -**Discrimination** -- Discriminatory jokes and language -- Posting or threatening to post others' personally identifying information ("doxing") -- Advocating for, or encouraging, any of the above behaviour -- Microaggressions — subtle, often unintentional, discriminatory comments or actions - -**Professional Misconduct** -- Publishing others' private information without explicit permission -- Misrepresenting affiliation or contributions -- Plagiarism or claiming credit for others' work -- Retaliating against anyone who reports a Code of Conduct violation -- Other conduct which could reasonably be considered inappropriate in a professional setting - -### Grey Areas - -Some situations require judgement. When uncertain: - -- **Intent vs Impact**: Good intentions do not excuse harmful impact. Focus on making things right. -- **Power Dynamics**: Those with more power (maintainers, employers, experienced contributors) must be especially mindful of their impact. -- **Cultural Differences**: What's acceptable varies by culture. When in doubt, err on the side of caution and ask. -- **Humour**: Jokes at others' expense are rarely funny to everyone. Punch up, not down. - ---- - -## Scope - -This Code of Conduct applies within all community spaces, including: - -**Online Spaces** -- Repository discussions, issues, and pull/merge requests -- Project chat channels (Matrix, Discord, Slack, IRC) -- Mailing lists and forums -- Social media when representing the project -- Video calls and virtual meetings - -**In-Person Spaces** -- Conferences, meetups, and events -- Workshops and training sessions -- Any gathering where you represent the project - -**Representation** -This Code of Conduct also applies when an individual is officially representing the community in public spaces. Examples include: - -- Using an official project email address -- Posting via an official social media account -- Acting as an appointed representative at an event -- Speaking on behalf of the project - ---- - -## Enforcement - -### Reporting - -If you experience or witness unacceptable behaviour, or have any other concerns, please report it as soon as possible. - -**How to Report** - -| Method | Details | Best For | -|--------|---------|----------| -| **Email** | {{CONDUCT_EMAIL}} | Detailed reports, sensitive matters | -| **Private Message** | Contact any maintainer directly | Quick questions, minor issues | -| **Anonymous Form** | [Link to form if available] | When you need anonymity | - -**What to Include** - -- Your contact information (unless anonymous) -- Names/usernames of those involved -- Description of what happened -- When and where it occurred -- Any witnesses -- Any supporting evidence (screenshots, links) -- How you would like us to respond (if you have a preference) - -**What Happens Next** - -1. You will receive acknowledgment within **{{RESPONSE_TIME}}** -2. The {{CONDUCT_TEAM}} will review the report -3. We may ask for additional information -4. We will determine appropriate action -5. We will inform you of the outcome (respecting others' privacy) - -### Confidentiality - -All reports will be handled with discretion: - -- Reporter identity is protected by default -- Details are shared only with those who need to know -- We will ask before naming you in any communication -- Anonymous reports are accepted and investigated - -### Conflicts of Interest - -If a {{CONDUCT_TEAM}} member is involved in an incident: - -- They will recuse themselves from the process -- Another maintainer or external party will handle the report -- We will disclose any potential conflicts - ---- - -## Enforcement Guidelines - -The {{CONDUCT_TEAM}} will follow these guidelines in determining consequences: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behaviour deemed unprofessional or unwelcome. - -**Consequence**: A private, written warning providing clarity around the nature of the violation and an explanation of why the behaviour was inappropriate. A public apology may be requested. - -**Duration**: Immediate - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of actions. - -**Consequence**: A warning with consequences for continued behaviour. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. - -**Duration**: 1-4 weeks - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including sustained inappropriate behaviour. - -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. - -**Duration**: 1-6 months - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behaviour, harassment of an individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the community. - -**Duration**: Permanent (with appeal rights after 12 months) - -### Enforcement Across Perimeters - -For contributors with elevated access (Perimeter 2 or 1): - -| Level | Additional Consequence | -|-------|----------------------| -| Correction | Noted in contributor record | -| Warning | Access privileges may be temporarily reduced | -| Temporary Ban | Access reduced to Perimeter 3 for ban duration | -| Permanent Ban | All access revoked | - ---- - -## Appeals - -If you believe an enforcement decision was made in error: - -1. **Wait 7 days** after the decision (cooling-off period) -2. **Email** {{CONDUCT_EMAIL}} with subject line "Appeal: [Original Report ID]" -3. **Explain** why you believe the decision should be reconsidered -4. **Provide** any new information not previously available - -**Appeals Process** - -- Appeals are reviewed by a different {{CONDUCT_TEAM}} member than the original -- You will receive a response within 14 days -- The appeals decision is final -- You may only appeal once per incident - -**Grounds for Appeal** - -- Procedural errors in the original investigation -- New evidence not previously available -- Disproportionate response to the violation -- Misunderstanding of facts - ---- - -## Supporting Those Who Report - -We are committed to supporting those who report violations: - -**We Will** -- Believe and take all reports seriously -- Respect your privacy and confidentiality preferences -- Keep you informed of progress (if you wish) -- Take steps to protect you from retaliation -- Provide resources if you need support - -**We Will Not** -- Require you to confront the person directly -- Dismiss reports without investigation -- Reveal your identity without consent -- Tolerate retaliation against reporters -- Rush you to make decisions - ---- - -## Prevention - -Beyond enforcement, we actively work to prevent issues: - -**Onboarding** -- All contributors are expected to read this Code of Conduct -- Perimeter 2 applicants must confirm they've read and understood it -- Maintainers receive additional training on enforcement - -**Culture** -- We model the behaviour we expect -- We intervene early when we see potential issues -- We thank people for positive contributions -- We create opportunities for diverse voices - -**Review** -- This Code of Conduct is reviewed annually -- Community feedback is welcomed -- Changes are communicated clearly - ---- - -## Acknowledgments - -This Code of Conduct is adapted from: - -- [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1 -- [Django Code of Conduct](https://www.djangoproject.com/conduct/) -- [Rust Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct) -- [Python Community Code of Conduct](https://www.python.org/psf/conduct/) - -We thank these communities for their leadership in creating welcoming spaces. - ---- - -## Questions? - -If you have questions about this Code of Conduct: - -- Open a [Discussion](https://github.com/hyperpolymath/nextgen-databases/discussions) (for general questions) -- Email {{CONDUCT_EMAIL}} (for private questions) -- Contact any maintainer directly - ---- - -## Summary - -**Be kind. Be respectful. Be collaborative.** - -We're all here because we care about this project. Let's make it a place where everyone can do their best work. - ---- - -Last updated: 2026 · Based on Contributor Covenant 2.1 diff --git a/lithoglyph/glyphbase/COLLABORATION-COMPLETE.md b/lithoglyph/glyphbase/COLLABORATION-COMPLETE.md deleted file mode 100644 index 593f9748..00000000 --- a/lithoglyph/glyphbase/COLLABORATION-COMPLETE.md +++ /dev/null @@ -1,220 +0,0 @@ -# Real-Time Collaboration Features - Complete - -## Summary - -Tasks #15, #16, and #17 are **COMPLETE ✅** - -## Task #15: Real-Time Collaboration with Yjs - -**Created:** -- `src/bindings/Yjs.res` - ReScript bindings for Yjs CRDT library -- `src/stores/CollaborationStore.res` - Collaboration state management - -**Features:** -- ✅ Yjs document creation and management -- ✅ Y.Map for collaborative cell updates (CRDT-based) -- ✅ Y.Array for collaborative row operations -- ✅ Y.Text for collaborative text editing -- ✅ Transaction support for atomic updates -- ✅ Event observation for remote changes -- ✅ Awareness protocol for cursor/presence tracking -- ✅ WebSocket provider integration (stub) - -**Architecture:** -``` -Grid Component - ↓ -CollaborationStore - ↓ -Yjs Bindings (ReScript) - ↓ -Yjs Library (JavaScript) - ↓ -WebSocket Provider - ↓ -Yjs Sync Server -``` - -## Task #16: Live Cursors and Presence Indicators - -**Created:** -- `src/components/LiveCursors.res` - Animated cursor indicators -- `src/components/PresenceIndicators.res` - Online user avatars -- `src/styles/collaboration.css` - Collaboration UI styles - -**Features:** -- ✅ Real-time cursor position tracking -- ✅ User color-coded cursors with name labels -- ✅ Animated cursor transitions -- ✅ Presence avatars showing online users -- ✅ User count and status display -- ✅ Connection status indicators (connected/connecting/disconnected) -- ✅ Active users panel -- ✅ Collaborative editing cell highlights - -**Visual Features:** -- Pulse animation for cursors -- Fade-in animations for presence avatars -- Hover effects on avatars -- Color-coded user indicators -- Status dots with pulse animation - -## Task #17: Cell Comments and @Mentions - -**Created:** -- `src/components/CellComments.res` - Comments UI -- `src/stores/CommentsStore.res` - Comments data management -- `src/styles/comments.css` - Comments UI styles - -**Features:** -- ✅ Cell-level comments (threaded per cell) -- ✅ @mention detection and autocomplete -- ✅ Mention extraction from comment text -- ✅ User mention notifications -- ✅ Comment count badges on cells -- ✅ Rich comment display with author info -- ✅ Timestamp display -- ✅ Keyboard shortcuts (Ctrl+Enter to submit) -- ✅ Comment CRUD operations (Create, Read, Update, Delete) - -**Data Model:** -```rescript -type comment = { - id: string, - rowId: string, - fieldId: string, - author: string, - authorId: string, - content: string, - mentions: array, // Extracted @mentions - createdAt: Date.t, - updatedAt: Date.t, -} -``` - -## Integration Points - -### CollaborationStore API - -```rescript -// Initialize collaboration -initCollaboration( - tableId, - userId, - userName, - ~wsUrl="ws://localhost:1234", - ~onConnected, - ~onSynced, - ~onDisconnected -) - -// Update cell collaboratively -updateCellCollab(rowId, fieldId, value) - -// Observe changes from other users -observeCellChanges(onCellChange) - -// Update cursor position -updateCursor(rowId, fieldId) - -// Get active users -getActiveUsers() - -// Disconnect -disconnectCollaboration() -``` - -### CommentsStore API - -```rescript -// Get comments for a cell -getCellComments(rowId, fieldId) - -// Add a comment -addComment(rowId, fieldId, content, authorId, authorName) - -// Get user mentions -getUserMentions(userId) - -// Get comment count -getCellCommentCount(rowId, fieldId) - -// Update/delete comments -updateComment(commentId, newContent) -deleteComment(commentId) -``` - -## Styling and UX - -**Collaboration.css Features:** -- Animated cursors with pulse effect -- Smooth cursor transitions -- Presence avatars with hover effects -- Connection status with visual feedback -- Active users panel styling - -**Comments.css Features:** -- Comments panel slide-in animation -- Mention highlighting -- Comment badges on cells -- Mention autocomplete dropdown -- Rich text formatting for mentions - -## Testing Plan - -1. **Yjs Sync:** - - Open same table in two browser windows - - Edit cells in one window - - Verify changes appear in other window - - Test conflict resolution (simultaneous edits) - -2. **Cursors:** - - Move cursor between cells - - Verify cursor appears in other clients - - Test cursor hiding when user disconnects - -3. **Presence:** - - Join with multiple users - - Verify all users shown in presence indicators - - Test user disconnection handling - -4. **Comments:** - - Add comment to cell - - Use @mentions - - Verify mention extraction - - Test comment persistence - -## WebSocket Server Required - -For full functionality, deploy a Yjs WebSocket server: - -```bash -npm install -g y-websocket -PORT=1234 npx y-websocket -``` - -Or use hosted Yjs sync servers like: -- [Yjs WebSocket Server](https://github.com/yjs/y-websocket) -- [Hocuspocus](https://tiptap.dev/hocuspocus) - -## Next Steps - -1. Deploy Yjs WebSocket server -2. Wire up CollaborationStore to Grid component -3. Add LiveCursors and PresenceIndicators to main layout -4. Implement CellComments panel toggle -5. Add comment notifications for @mentions -6. Store comments in Lithoglyph database -7. Add comment reactions (emoji) -8. Add comment editing history - -## Performance Considerations - -- Yjs uses CRDTs for efficient conflict resolution -- Awareness updates are throttled (100ms default) -- Comment loading lazy-loaded per cell -- Cursor updates batched for performance - -## License - -PMPL-1.0-or-later (Palimpsest License) diff --git a/lithoglyph/glyphbase/CONTRIBUTING.md b/lithoglyph/glyphbase/CONTRIBUTING.md deleted file mode 100644 index bd6f5a5e..00000000 --- a/lithoglyph/glyphbase/CONTRIBUTING.md +++ /dev/null @@ -1,116 +0,0 @@ -# Clone the repository -git clone https://github.com/hyperpolymath/nextgen-databases.git -cd nextgen-databases - -# Using Nix (recommended for reproducibility) -nix develop - -# Or using toolbox/distrobox -toolbox create nextgen-databases-dev -toolbox enter nextgen-databases-dev -# Install dependencies manually - -# Verify setup -just check # or: cargo check / mix compile / etc. -just test # Run test suite -``` - -### Repository Structure -``` -nextgen-databases/ -├── src/ # Source code (Perimeter 1-2) -├── lib/ # Library code (Perimeter 1-2) -├── extensions/ # Extensions (Perimeter 2) -├── plugins/ # Plugins (Perimeter 2) -├── tools/ # Tooling (Perimeter 2) -├── docs/ # Documentation (Perimeter 3) -│ ├── architecture/ # ADRs, specs (Perimeter 2) -│ └── proposals/ # RFCs (Perimeter 3) -├── examples/ # Examples (Perimeter 3) -├── spec/ # Spec tests (Perimeter 3) -├── tests/ # Test suite (Perimeter 2-3) -├── .well-known/ # Protocol files (Perimeter 1-3) -├── .github/ # GitHub config (Perimeter 1) -│ ├── ISSUE_TEMPLATE/ -│ └── workflows/ -├── CHANGELOG.md -├── CODE_OF_CONDUCT.md -├── CONTRIBUTING.md # This file -├── GOVERNANCE.md -├── LICENSE -├── MAINTAINERS.md -├── README.adoc -├── SECURITY.md -├── flake.nix # Nix flake (Perimeter 1) -└── Justfile # Task runner (Perimeter 1) -``` - ---- - -## How to Contribute - -### Reporting Bugs - -**Before reporting**: -1. Search existing issues -2. Check if it's already fixed in `main` -3. Determine which perimeter the bug affects - -**When reporting**: - -Use the [bug report template](.github/ISSUE_TEMPLATE/bug_report.md) and include: - -- Clear, descriptive title -- Environment details (OS, versions, toolchain) -- Steps to reproduce -- Expected vs actual behaviour -- Logs, screenshots, or minimal reproduction - -### Suggesting Features - -**Before suggesting**: -1. Check the [roadmap](ROADMAP.md) if available -2. Search existing issues and discussions -3. Consider which perimeter the feature belongs to - -**When suggesting**: - -Use the [feature request template](.github/ISSUE_TEMPLATE/feature_request.md) and include: - -- Problem statement (what pain point does this solve?) -- Proposed solution -- Alternatives considered -- Which perimeter this affects - -### Your First Contribution - -Look for issues labelled: - -- [`good first issue`](https://github.com/hyperpolymath/nextgen-databases/labels/good%20first%20issue) — Simple Perimeter 3 tasks -- [`help wanted`](https://github.com/hyperpolymath/nextgen-databases/labels/help%20wanted) — Community help needed -- [`documentation`](https://github.com/hyperpolymath/nextgen-databases/labels/documentation) — Docs improvements -- [`perimeter-3`](https://github.com/hyperpolymath/nextgen-databases/labels/perimeter-3) — Community sandbox scope - ---- - -## Development Workflow - -### Branch Naming -``` -docs/short-description # Documentation (P3) -test/what-added # Test additions (P3) -feat/short-description # New features (P2) -fix/issue-number-description # Bug fixes (P2) -refactor/what-changed # Code improvements (P2) -security/what-fixed # Security fixes (P1-2) -``` - -### Commit Messages - -We follow [Conventional Commits](https://www.conventionalcommits.org/): -``` -(): - -[optional body] - -[optional footer] diff --git a/lithoglyph/glyphbase/Containerfile b/lithoglyph/glyphbase/Containerfile deleted file mode 100644 index 00c3b22b..00000000 --- a/lithoglyph/glyphbase/Containerfile +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Glyphbase Docker Image -# Multi-stage build for minimal final image - -# Stage 1: Build UI -FROM docker.io/denoland/deno:2.1.4 AS ui-builder - -WORKDIR /build/ui - -# Copy UI dependencies -COPY ui/deno.json ui/deno.lock ./ -COPY ui/package.json ui/rescript.json ./ - -# Install dependencies -RUN deno cache deno.json - -# Copy UI source -COPY ui/ . - -# Build UI -RUN deno task build - -# Stage 2: Build Server -FROM docker.io/hexpm/gleam:1.7.1-erlang-27.2.1 AS server-builder - -WORKDIR /build/server - -# Copy server dependencies -COPY server/gleam.toml server/manifest.toml* ./ - -# Download dependencies -RUN gleam deps download - -# Copy server source -COPY server/src ./src -COPY server/test ./test - -# Build server -RUN gleam build --target erlang - -# Stage 3: Runtime -FROM docker.io/hexpm/erlang:27.2.1-alpine - -WORKDIR /app - -# Install runtime dependencies -RUN apk add --no-cache libstdc++ openssl - -# Copy built server from builder -COPY --from=server-builder /build/server/build /app/build - -# Copy built UI from builder -COPY --from=ui-builder /build/ui/dist /app/public - -# Create data directory -RUN mkdir -p /data && chmod 777 /data - -# Environment variables -ENV PORT=4000 -ENV DATABASE_PATH=/data -ENV GLEAM_ERLANG_PATH=/app/build - -# Expose port -EXPOSE 4000 - -# Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://localhost:4000/health || exit 1 - -# Run server -CMD ["gleam", "run"] diff --git a/lithoglyph/glyphbase/INSTALL.md b/lithoglyph/glyphbase/INSTALL.md deleted file mode 100644 index c948c22c..00000000 --- a/lithoglyph/glyphbase/INSTALL.md +++ /dev/null @@ -1,397 +0,0 @@ -# Glyphbase Installation Guide - -**Carve your data in stone** - Multiple installation options for every use case. - -## Quick Start (Recommended) - -### Option 1: Docker (Easiest) - -```bash -# Pull the latest image -docker pull ghcr.io/hyperpolymath/glyphbase:latest - -# Run Glyphbase -docker run -p 4000:4000 -v ./data:/data ghcr.io/hyperpolymath/glyphbase:latest - -# Open http://localhost:4000 -``` - -### Option 2: Docker Compose (Production-Ready) - -```bash -# Clone the repository -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase - -# Start with docker-compose -docker-compose up -d - -# Open http://localhost:4000 -``` - -### Option 3: From Source (Development) - -**Prerequisites:** -- [Gleam](https://gleam.run) 1.7+ -- [Deno](https://deno.land) 2.0+ -- [Lithoglyph](https://github.com/hyperpolymath/lithoglyph) database -- Erlang/OTP 27+ - -```bash -# Clone the repository -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase - -# Start development servers -just dev - -# Or manually: -cd server && gleam run & -cd ui && deno task dev - -# Open http://localhost:4000 -``` - -## Detailed Installation Options - -### 1. Docker (Single Container) - -**Pros:** Easiest setup, works everywhere, no dependencies -**Cons:** Less control over configuration - -```bash -# Basic run -docker run -p 4000:4000 ghcr.io/hyperpolymath/glyphbase:latest - -# With persistent data -docker run -p 4000:4000 \ - -v $PWD/data:/data \ - ghcr.io/hyperpolymath/glyphbase:latest - -# With custom configuration -docker run -p 4000:4000 \ - -v $PWD/data:/data \ - -e SECRET_KEY=your-secret-key \ - -e PORT=8080 \ - ghcr.io/hyperpolymath/glyphbase:latest -``` - -### 2. Docker Compose (Multi-Container) - -**Pros:** Production-ready, includes Lithoglyph database, easy scaling -**Cons:** Requires docker-compose - -```bash -# Clone repository -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase - -# Start services -docker-compose up -d - -# Check logs -docker-compose logs -f - -# Stop services -docker-compose down - -# With Lithoglyph standalone database -docker-compose --profile full up -d -``` - -**Configuration:** - -Create `.env` file: - -```env -SECRET_KEY=your-production-secret-key -PORT=4000 -DATABASE_PATH=/data -``` - -### 3. Podman (Docker Alternative) - -**Pros:** Rootless, more secure, compatible with Docker -**Cons:** Slightly different commands - -```bash -# Pull image -podman pull ghcr.io/hyperpolymath/glyphbase:latest - -# Run with podman -podman run -p 4000:4000 -v ./data:/data:Z ghcr.io/hyperpolymath/glyphbase:latest - -# Or use Justfile -just docker-build # Uses podman -just docker-run -``` - -### 4. From Source (Development & Contributors) - -**Pros:** Full control, latest features, contribution-ready -**Cons:** Requires multiple dependencies - -#### Prerequisites - -**Install Gleam:** -```bash -# Linux/macOS -curl -fsSL https://gleam.run/install.sh | sh - -# Or via asdf -asdf plugin add gleam -asdf install gleam latest -``` - -**Install Deno:** -```bash -# Linux/macOS -curl -fsSL https://deno.land/install.sh | sh - -# Or via asdf -asdf plugin add deno -asdf install deno latest -``` - -**Install Lithoglyph:** -```bash -# See https://github.com/hyperpolymath/lithoglyph -git clone https://github.com/hyperpolymath/lithoglyph -cd lithoglyph -# Follow lithoglyph installation instructions -``` - -#### Build & Run - -```bash -# Clone Glyphbase -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase - -# Install dependencies (automatic) -cd server && gleam deps download -cd ../ui && deno cache deno.json - -# Development mode (hot reload) -just dev - -# Build for production -just build - -# Run tests -just test - -# Format & lint -just fmt -just lint -``` - -#### Development Workflow - -```bash -# Start UI dev server (port 3000) -just dev-ui - -# Start backend server (port 4000) -just dev-server - -# Run both in parallel -just dev -``` - -**Ports:** -- UI (dev): `http://localhost:3000` -- Server: `http://localhost:4000` -- API: `http://localhost:4000/api` - -### 5. Self-Hosting on VPS - -**Pros:** Full control, custom domain, professional deployment -**Cons:** Requires server management skills - -#### Using Docker on VPS - -```bash -# SSH into your server -ssh user@your-server.com - -# Install Docker -curl -fsSL https://get.docker.com | sh - -# Run Glyphbase -docker run -d --restart unless-stopped \ - -p 80:4000 \ - -v /var/lib/glyphbase:/data \ - --name glyphbase \ - ghcr.io/hyperpolymath/glyphbase:latest - -# Set up reverse proxy (nginx/caddy) -# Point your domain to the server -# Configure SSL with Let's Encrypt -``` - -#### With Caddy (Automatic HTTPS) - -**Caddyfile:** -``` -glyphbase.yourdomain.com { - reverse_proxy localhost:4000 -} -``` - -```bash -caddy run --config Caddyfile -``` - -### 6. Kubernetes Deployment - -**Pros:** Enterprise-scale, auto-scaling, high availability -**Cons:** Complex setup - -```yaml -# glyphbase-deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: glyphbase -spec: - replicas: 3 - selector: - matchLabels: - app: glyphbase - template: - metadata: - labels: - app: glyphbase - spec: - containers: - - name: glyphbase - image: ghcr.io/hyperpolymath/glyphbase:latest - ports: - - containerPort: 4000 - env: - - name: DATABASE_PATH - value: /data - volumeMounts: - - name: data - mountPath: /data - volumes: - - name: data - persistentVolumeClaim: - claimName: glyphbase-data ---- -apiVersion: v1 -kind: Service -metadata: - name: glyphbase -spec: - selector: - app: glyphbase - ports: - - port: 80 - targetPort: 4000 - type: LoadBalancer -``` - -```bash -kubectl apply -f glyphbase-deployment.yaml -``` - -## Configuration - -### Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `PORT` | `4000` | HTTP server port | -| `DATABASE_PATH` | `./data` | Lithoglyph data directory | -| `SECRET_KEY` | *(required)* | Session encryption key (production) | -| `LOG_LEVEL` | `info` | Logging level (debug, info, warn, error) | -| `MAX_UPLOAD_SIZE` | `100MB` | Maximum file upload size | - -### Data Persistence - -Glyphbase stores all data in the Lithoglyph database directory. - -**Important:** Always mount a volume to `/data` for persistence: - -```bash -docker run -v ./data:/data glyphbase # Persists to ./data -docker run -v /var/lib/glyphbase:/data glyphbase # Persists to /var/lib/glyphbase -``` - -## Troubleshooting - -### Port Already in Use - -```bash -# Check what's using port 4000 -lsof -i :4000 - -# Use a different port -docker run -p 8080:4000 glyphbase -``` - -### Database Connection Errors - -```bash -# Check Lithoglyph is running -curl http://localhost:5432/health - -# Check data directory permissions -ls -la ./data -chmod 777 ./data # or appropriate permissions -``` - -### Container Won't Start - -```bash -# Check logs -docker logs glyphbase - -# Check health -docker inspect glyphbase | grep Health -``` - -## Upgrading - -### Docker - -```bash -# Pull latest image -docker pull ghcr.io/hyperpolymath/glyphbase:latest - -# Stop old container -docker stop glyphbase && docker rm glyphbase - -# Start new container (data persists in volume) -docker run -p 4000:4000 -v ./data:/data ghcr.io/hyperpolymath/glyphbase:latest -``` - -### From Source - -```bash -cd glyphbase -git pull origin main -gleam deps download -deno cache deno.json -just build -``` - -## Getting Help - -- **Documentation:** https://glyphbase.lithoglyph.org/docs -- **Community:** https://github.com/hyperpolymath/glyphbase/discussions -- **Issues:** https://github.com/hyperpolymath/glyphbase/issues -- **Email:** support@lithoglyph.org - -## Next Steps - -1. **Quick Start Guide:** See [QUICKSTART.md](./QUICKSTART.md) -2. **User Guide:** See [docs/USER-GUIDE.adoc](./docs/USER-GUIDE.adoc) -3. **API Documentation:** https://docs.lithoglyph.org/api -4. **Contributing:** See [CONTRIBUTING.md](./CONTRIBUTING.md) - ---- - -**Welcome to Glyphbase!** Carve your data in stone. 🪨 diff --git a/lithoglyph/glyphbase/Justfile b/lithoglyph/glyphbase/Justfile deleted file mode 100644 index ee3d2289..00000000 --- a/lithoglyph/glyphbase/Justfile +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Glyphbase - Build Commands - -# Default recipe -default: - @just --list - -# Development - run both UI and server -dev: - @echo "Starting Glyphbase development servers..." - @just dev-server & - @just dev-ui - -# UI development server -dev-ui: - cd ui && deno task dev - -# Server development -dev-server: - cd server && gleam run - -# Build everything -build: - @just build-ui - @just build-server - -# Build UI -build-ui: - cd ui && deno task build - -# Build server -build-server: - cd server && gleam build - -# Run all tests -test: - @just test-ui - @just test-server - -# UI tests -test-ui: - cd ui && deno task test - -# Server tests -test-server: - cd server && gleam test - -# Format all code -fmt: - cd ui && deno task fmt - cd server && gleam format - -# Lint all code -lint: - cd ui && deno task lint - -# Clean build artifacts -clean: - rm -rf ui/dist - rm -rf server/build - -# Docker build -docker-build: - podman build -t glyphbase:latest . - -# Docker run -docker-run: - podman run -p 4000:4000 -v ./data:/data glyphbase:latest - -# Full docker-compose stack -up: - podman-compose up - -# Stop docker-compose stack -down: - podman-compose down diff --git a/lithoglyph/glyphbase/LICENSE b/lithoglyph/glyphbase/LICENSE deleted file mode 100644 index ec540b34..00000000 --- a/lithoglyph/glyphbase/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: MPL-2.0 -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/lithoglyph/glyphbase/LITHOGLYPH-INTEGRATION.md b/lithoglyph/glyphbase/LITHOGLYPH-INTEGRATION.md deleted file mode 100644 index b27a50ab..00000000 --- a/lithoglyph/glyphbase/LITHOGLYPH-INTEGRATION.md +++ /dev/null @@ -1,33 +0,0 @@ -# Lithoglyph Integration Status - -**Date:** 2026-02-04 -**Status:** ✅ COMPLETE - M10 PoC Integration Working - -## Summary - -Glyphbase now successfully integrates with Lithoglyph via Rustler NIF. All 9 NIF functions work correctly from both Erlang and Gleam. - -## Test Results - -``` -=== Lithoglyph NIF Test (Glyphbase) === -Test 1: Version {1,0,0} ✓ -Test 2: Database opened ✓ -Test 3: Transaction started ✓ -Test 4: Operation applied, block ID: [0,0,0,0,0,0,0,1] ✓ -Test 5: Transaction committed ✓ -Test 6: Schema: CBOR empty map ✓ -Test 7: Journal: CBOR empty array ✓ -Test 8: Database closed ✓ -=== All tests passed! === -``` - -## Next Steps - M11 HTTP API - -Priority 3: Create HTTP API wrapper for Lithoglyph-Geo and Lithoglyph-Analytics access. - ---- - -**Completed:** 2026-02-04 (M10 Day 3) -**Technology:** Rust (Rustler 0.35) + Gleam + Erlang -EOF diff --git a/lithoglyph/glyphbase/MAINTAINERS.adoc b/lithoglyph/glyphbase/MAINTAINERS.adoc deleted file mode 100644 index 48d97817..00000000 --- a/lithoglyph/glyphbase/MAINTAINERS.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Maintainers -:toc: preamble - -This document lists the maintainers of this project and their responsibilities. - -== Current Maintainers - -[cols="2,3,2",options="header"] -|=== -| Name | Role | Contact - -| Jonathan D.A. Jewell -| Lead Maintainer -| https://github.com/hyperpolymath[@hyperpolymath] -|=== - -== Responsibilities - -Maintainers are responsible for: - -* Reviewing and merging pull requests -* Triaging issues and feature requests -* Ensuring code quality and security standards -* Managing releases and versioning -* Upholding the project's code of conduct - -== Becoming a Maintainer - -Contributors who demonstrate: - -* Consistent, high-quality contributions -* Understanding of the project's goals and standards -* Constructive participation in discussions -* Commitment to the project's long-term health - -May be invited to become maintainers at the discretion of existing maintainers. - -== Decision Making - -* Routine decisions (bug fixes, minor improvements) can be made by any maintainer -* Significant changes require discussion and consensus among maintainers -* Breaking changes or major features should be discussed in issues before implementation - -== Contact - -For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/lithoglyph/glyphbase/PROVEN-INTEGRATION.md b/lithoglyph/glyphbase/PROVEN-INTEGRATION.md deleted file mode 100644 index 85b40034..00000000 --- a/lithoglyph/glyphbase/PROVEN-INTEGRATION.md +++ /dev/null @@ -1,171 +0,0 @@ -# Proven Library Integration - Complete - -## Summary - -Task #14 (Add Proven library integration with Idris2 proofs) is **COMPLETE ✅** - -## What Was Done - -1. **Copied Proven ReScript bindings** to `ui/src/lib/proven/` - - ProvenResult.res - Result type for error handling - - ProvenSafeString.res - Formally verified string operations - - ProvenSafeUrl.res - URL parsing with formal proofs - - ProvenSafeJson.res - Safe JSON manipulation - -2. **Created ProvenFieldValidation module** for cell validation -3. **Updated license headers** to PMPL-1.0-or-later - -## Architecture - -``` -Glyphbase Application (ReScript) - ↓ -ProvenFieldValidation (validation layer) - ↓ -Proven ReScript Bindings (type-safe wrappers) - ↓ -Proven JavaScript Bindings (FFI glue) - ↓ -Zig FFI Bridge (C ABI compatibility) - ↓ -Idris2 ABI (formally verified implementations) - ↓ -MATHEMATICAL PROOFS ✓ -``` - -## Formal Guarantees - -The Proven library provides **compile-time mathematical proofs** that operations cannot crash: - -### ProvenSafeString -- ✅ **Bounds checking**: `charAt()` and `substring()` prove indices are valid -- ✅ **No crashes**: String operations guaranteed total (always terminate) -- ✅ **Encoding safety**: Handles Unicode correctly - -### ProvenSafeUrl -- ✅ **Well-formedness**: URLs proven to match RFC 3986 -- ✅ **Injection prevention**: No URL injection attacks possible -- ✅ **Type safety**: Invalid URLs rejected at validation boundary - -### ProvenSafeJson -- ✅ **Parse safety**: JSON parsing cannot throw exceptions -- ✅ **Type validation**: Access operations check types -- ✅ **Path safety**: Nested access guaranteed safe - -## Validation Module API - -```rescript -open ProvenFieldValidation - -// Validate text field -let result = validateText("Hello", ~maxLength=Some(100)) - -// Validate URL field -let result = validateUrl("https://example.com") - -// Validate email field -let result = validateEmail("user@example.com") - -// Validate cell value based on field type -let result = validateCellValue( - Text, - TextValue("sample"), - ~required=true -) - -// Batch validate all fields -let errors = validateFields(fields, cells) -// Returns array<(fieldId, errorMessage)> -``` - -## Integration Points - -The ProvenFieldValidation module can be used in: - -1. **Cell editing** - Validate before saving to database -2. **Form submission** - Validate all fields before API call -3. **Import/export** - Validate data integrity -4. **API boundaries** - Validate incoming data - -## Example Usage in Grid - -```rescript -// In Grid component when cell is edited -let handleCellUpdate = (rowId, fieldId, newValue) => { - let field = table.fields->Array.find(f => f.id == fieldId) - - switch field { - | Some(f) => { - let validationResult = ProvenFieldValidation.validateCellValue( - f.fieldType, - newValue, - ~required=f.required - ) - - switch validationResult { - | Valid => { - // Save to database - updateCell(rowId, fieldId, newValue) - } - | Invalid(msg) => { - // Show error to user - Console.error(`Validation failed: ${msg}`) - } - } - } - | None => () - } -} -``` - -## Benefits Over Regular Validation - -| Regular Validation | Proven Validation | -|-------------------|------------------| -| Trust the author | Mathematical proof | -| Runtime crashes possible | Cannot crash (totality) | -| String bounds unchecked | Bounds proven at compile-time | -| URL parsing throws | Parse result always valid | -| Manual error handling | Exhaustive by construction | - -## Performance Considerations - -- **FFI overhead**: Crossing from ReScript → JS → Zig → Idris2 has cost -- **Tradeoff**: Correctness over speed (deliberate design choice) -- **Mitigation**: Batch operations when possible to reduce crossings -- **Use case**: Critical validation where correctness > performance - -## What's Proven vs Not Proven - -**Proven (Idris2 verified):** -- String operations (charAt, substring, trim, etc.) -- URL parsing and validation -- JSON parsing and access -- Result type guarantees - -**Not Proven (standard ReScript):** -- UI rendering -- State management (Jotai) -- Grid component logic -- API client calls - -The Proven library targets **data validation boundaries** where correctness is critical. - -## Next Steps - -1. Wire up ProvenFieldValidation to Grid cell editing -2. Add validation to Form component -3. Use ProvenSafeJson for API request/response handling -4. Add ProvenSafeDateTime when available for date field validation -5. Consider ProvenSafeMath for numeric cell operations - -## License - -The Proven library declares MPL-2.0-or-later for platform compatibility. -Glyphbase integration code uses PMPL-1.0-or-later (Palimpsest license). - -## References - -- [Proven Library](https://github.com/hyperpolymath/proven) - Formally verified Idris2 library -- [Idris2 Documentation](https://idris2.readthedocs.io/) - Dependent types and totality checking -- [ABI/FFI Universal Standard](../server/ffi/zig/ABI-FFI-README.md) - Architecture documentation diff --git a/lithoglyph/glyphbase/QUICKSTART.md b/lithoglyph/glyphbase/QUICKSTART.md deleted file mode 100644 index 6b61e9f4..00000000 --- a/lithoglyph/glyphbase/QUICKSTART.md +++ /dev/null @@ -1,109 +0,0 @@ -# Glyphbase Quick Start - -**Get up and running with Glyphbase in 5 minutes.** - -## Prerequisites - -- [Gleam](https://gleam.run) 1.0+ -- [Lithoglyph](https://github.com/hyperpolymath/lithoglyph) database -- Erlang/OTP 26+ - -## Installation - -```bash -# 1. Clone Glyphbase -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase/server - -# 2. Install dependencies -gleam deps download - -# 3. Start the server -gleam run - -# 4. Open your browser -open http://localhost:4000 -``` - -## Your First Table - -1. Click **"+ New Table"** -2. Choose the **"Task Tracker"** template -3. Add a few tasks -4. Try different views: Grid → Kanban → Calendar -5. Click any cell to see its full provenance history - -## Import Example Data - -Load pre-made examples: - -```bash -# Task tracker -curl -X POST http://localhost:4000/api/import \ - -H "Content-Type: application/json" \ - -d @examples/task-tracker.json - -# Research papers -curl -X POST http://localhost:4000/api/import \ - -H "Content-Type: application/json" \ - -d @examples/research-papers.json -``` - -## Key Features to Try - -### 1. Provenance (Time Travel) -- Right-click any cell → **"View History"** -- See every change with who/when/why -- Click any point to restore - -### 2. PROMPT Scores -- Open the "Research Papers" table -- See quality scores for each paper -- Filter by `prompt_total >= 80` for high-quality papers only - -### 3. Multiple Views -- **Grid**: Traditional spreadsheet -- **Kanban**: Drag tasks across status columns -- **Calendar**: See tasks by due date -- **Gallery**: Visual cards with images - -### 4. Real-time Collaboration -- Open the same table in two browser windows -- Edit in one window → see live updates in the other -- No conflicts, ever (append-only journal) - -## Next Steps - -- Read the [User Guide](docs/USER-GUIDE.adoc) -- Explore the [API documentation](https://docs.lithoglyph.org/api) -- Join the [community discussions](https://github.com/hyperpolymath/glyphbase/discussions) - -## Self-Hosting - -Deploy Glyphbase on your own infrastructure: - -### Docker - -```bash -docker pull ghcr.io/hyperpolymath/glyphbase:latest -docker run -p 4000:4000 -v ./data:/data glyphbase -``` - -### From Source - -```bash -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase/server -gleam run -``` - -## Getting Help - -- 📖 [Full Documentation](https://docs.lithoglyph.org) -- 💬 [Community](https://github.com/hyperpolymath/glyphbase/discussions) -- 🐛 [Report Issues](https://github.com/hyperpolymath/glyphbase/issues) -- ✉️ support@lithoglyph.org - ---- - -**Welcome to Glyphbase!** Carve your data in stone. 🪨 diff --git a/lithoglyph/glyphbase/README.adoc b/lithoglyph/glyphbase/README.adoc deleted file mode 100644 index 829f7d0f..00000000 --- a/lithoglyph/glyphbase/README.adoc +++ /dev/null @@ -1,344 +0,0 @@ -image:https://img.shields.io/badge/License-MPL--2.0-blue.svg[License: PMPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] -image:https://img.shields.io/badge/Powered_by-Idris-purple.svg[Powered by Idris,link="https://www.idris-lang.org/"] -// SPDX-License-Identifier: CC-BY-SA-4.0 -// SPDX-FileCopyrightText: 2025 hyperpolymath - -= Glyphbase -:toc: -:toc-placement: preamble -:toclevels: 3 - -[.lead] -*Carve your data in stone.* - -_The open-source Airtable alternative with permanence, powered by formal verification._ - -== What is Glyphbase? - -Glyphbase is a spreadsheet-database hybrid built on link:https://github.com/hyperpolymath/lithoglyph[Lithoglyph]. Unlike traditional spreadsheet tools, every change is carved in stone - tracked with who made it, when, and why - and fully reversible forever. - -=== Key Features - -[cols="1,2"] -|=== -| Feature | Description - -| *Provenance by default* -| Every cell change records who, when, and why - not optional - -| *Full reversibility* -| Undo any change, forever, with mathematical proof - -| *Cloud-safe* -| Sync via Dropbox/GDrive without corruption (append-only) - -| *Self-hosted* -| Your data stays on your servers - -| *PROMPT scores* -| Research-grade data quality scoring - -| *Multiple views* -| Grid, Kanban, Calendar, Gallery, Form -|=== - -== Quick Start - -[source,bash] ----- -# Clone the repo -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase - -# Start the server (requires Gleam and Lithoglyph) -cd server && gleam run - -# Start the UI (requires Deno) -cd ../ui && deno task dev - -# Open in browser -open http://localhost:3000 ----- - -== Screenshots - -_Coming soon_ - -== Architecture - -[source,text] ----- -┌─────────────────────────────────────────────────────────────┐ -│ Browser UI (ReScript + React) │ -│ • rescript-dom-mounter (high-assurance rendering) │ -│ • Proven library integration (Idris formal verification) │ -│ Grid | Kanban | Calendar | Gallery | Form Builder │ -├─────────────────────────────────────────────────────────────┤ -│ Real-time Collaboration (WebSocket + Yjs CRDT) │ -├─────────────────────────────────────────────────────────────┤ -│ API Server (Gleam on BEAM) │ -│ REST + WebSocket + Automations │ -├─────────────────────────────────────────────────────────────┤ -│ Lithoglyph Engine │ -│ Provenance | Reversibility | PROMPT Scores │ -├─────────────────────────────────────────────────────────────┤ -│ Formal Verification Layer (Idris2) │ -│ Type-level guarantees | Correctness proofs │ -├─────────────────────────────────────────────────────────────┤ -│ Storage: Local | Dropbox | GDrive | S3 │ -└─────────────────────────────────────────────────────────────┘ ----- - -== Views - -=== Grid View -Spreadsheet-like interface with sorting, filtering, grouping, and column customization. - -=== Kanban View -Drag-and-drop cards between columns, grouped by any select field. - -=== Calendar View -Events displayed on a calendar, driven by date fields. - -=== Gallery View -Cards with images, perfect for visual content. - -=== Form View -Public or private forms for data collection, with automatic provenance. - -== Field Types - -[cols="1,1,2"] -|=== -| Type | Icon | Notes - -| Text | Aa | Single or multi-line -| Number | # | Integer or decimal, with units -| Select | ▼ | Single or multi-select -| Date | 📅 | Date, datetime, or date range -| Checkbox | ☑ | Boolean -| Link | 🔗 | Link to another table (foreign key) -| Attachment | 📎 | Files, images, documents -| Formula | fx | Computed from other fields -| Rollup | Σ | Aggregate linked records -| Lookup | 👁 | Pull field from linked record -| URL | 🌐 | Web links -| Email | ✉ | Email addresses -| Phone | ☎ | Phone numbers -| Rating | ⭐ | 1-5 stars -| Barcode | ▮▮▮ | QR codes, barcodes -|=== - -== Lithoglyph Superpowers - -These features are unique to Glyphbase because of the Lithoglyph backend: - -=== Provenance View -Click any cell to see its complete history: who changed it, when, and their stated rationale. - -=== Time Travel -View your entire base at any point in history. Compare snapshots. - -=== PROMPT Scores -Rate data quality on six dimensions: Provenance, Replicability, Objectivity, Methodology, Publication, Transparency. - -=== Proof Export -Generate audit-grade documentation with cryptographic proofs of data integrity. - -=== DOI Linking -Link records to canonical sources. DOI-linked records are immutable; edits create "play variants." - -== Automations - -[cols="1,1"] -|=== -| Triggers | Actions - -| Row created -| Send webhook - -| Field changed -| Send email - -| Form submitted -| Post to Slack - -| Scheduled (cron) -| Run script - -| PROMPT score below threshold -| Flag for review -|=== - -== Comparison - -[cols="1,1,1,1,1"] -|=== -| Feature | Airtable | NocoDB | Baserow | *Glyphbase* - -| Self-hosted -| ❌ -| ✓ -| ✓ -| ✓ - -| Open source -| ❌ -| ✓ -| ✓ -| ✓ - -| Provenance tracking -| ❌ -| ❌ -| ❌ -| ✓ *Built-in* - -| Full undo history -| Limited -| Limited -| Limited -| ✓ *Forever* - -| Cloud sync safe -| N/A -| ❌ -| ❌ -| ✓ *Append-only* - -| Data quality scores -| ❌ -| ❌ -| ❌ -| ✓ *PROMPT* - -| Offline-first -| ❌ -| ❌ -| ❌ -| ✓ - -| Your data ownership -| ❌ -| ✓ -| ✓ -| ✓ -|=== - -== Roadmap - -=== v0.1.0 - Core Grid ✓ _Complete_ -* [x] Create/delete bases and tables -* [x] Add/edit/delete rows -* [x] Core field types (text, number, date, select, checkbox) -* [x] Sort, filter, hide columns -* [x] Keyboard navigation -* [x] Column resizing -* [x] Search across all cells -* [x] Undo/redo with full history - -=== v0.2.0 - Views ✓ _Complete_ -* [x] Kanban view (drag-and-drop) -* [x] Calendar view (month/week/day) -* [x] Gallery view (grid/masonry) -* [x] Form builder (public forms) - -=== v0.3.0 - Backend Integration ✓ _Complete_ -* [x] Idris2 ABI definitions (formal proofs) -* [x] Zig FFI implementation (C-compatible) -* [x] NIF builds successfully (504KB shared library) -* [ ] Wire up Lithoglyph database (backend pending) -* [ ] Provenance tracking integration (backend pending) -* [ ] CBOR operation encoding (backend pending) - -=== v0.4.0 - Formal Verification ✓ _Complete_ -* [x] rescript-dom-mounter (high-assurance rendering) -* [x] Proven library integration (Idris2 proofs) -* [x] Type-level guarantees for cell data integrity -* [x] Formal proofs for critical operations - -=== v0.5.0 - Real-time Collaboration ✓ _Complete - UI Ready_ -* [x] Real-time collaboration (WebSocket + Yjs CRDT) -* [x] Live cursors and presence indicators -* [x] Cell comments and @mentions -* [x] 97/97 modules compiling successfully -* [x] Production build: 65.66 kB gzipped (exceptional!) -* [x] Preview server running and tested - -=== v0.6.0 - Automations _(Next Priority)_ -* [ ] Trigger/action system -* [ ] Webhooks -* [ ] Email notifications -* [ ] Scheduled tasks - -=== v0.5.0 - Lithoglyph Superpowers -* [ ] Provenance view -* [ ] Time travel UI -* [ ] PROMPT scoring -* [ ] Proof export - -== Tech Stack - -[cols="1,2"] -|=== -| Layer | Technology - -| UI Framework -| ReScript + React - -| High-Assurance Rendering -| link:https://github.com/hyperpolymath/rescript-dom-mounter[rescript-dom-mounter] - -| Formal Verification -| link:https://github.com/hyperpolymath/proven[Proven] (Idris2 proofs) - -| State Management -| Jotai - -| Grid Component -| Custom (accessibility-first) - -| Real-time -| WebSocket + Yjs - -| Backend -| Gleam on BEAM - -| Database -| Lithoglyph - -| ABI/FFI -| link:server/ABI-FFI-README.md[Idris2 ABI + Zig FFI] (formal verification) - -| Auth -| Magic link + OIDC - -| Deploy -| Podman / Docker -|=== - -== Related Projects - -* link:https://github.com/hyperpolymath/lith[Lithoglyph] - The narrative-first database engine -* link:https://github.com/hyperpolymath/gqldt[GQLdt] - Dependently-typed query language -* link:https://github.com/hyperpolymath/lith-studio[Lithoglyph Studio] - Admin GUI for Lithoglyph -* link:https://github.com/hyperpolymath/lith-debugger[Lithoglyph Debugger] - Proof-carrying recovery tool -* link:https://github.com/hyperpolymath/bofig[BoFIG] - Evidence graph for journalism -* link:https://github.com/hyperpolymath/zotero-lith[Zotero-Lithoglyph] - Reference manager - -== License - -PMPL-1.0-or-later - -== Contributing - -See link:CONTRIBUTING.adoc[CONTRIBUTING.adoc] for guidelines. - -== Acknowledgements - -Inspired by: - -* link:https://airtable.com[Airtable] - The UX gold standard -* link:https://github.com/nocodb/nocodb[NocoDB] - Open source Airtable alternative -* link:https://github.com/bram2w/baserow[Baserow] - Self-hosted database -* link:https://github.com/gristlabs/grist-core[Grist] - Spreadsheet with formulas diff --git a/lithoglyph/glyphbase/ROADMAP.adoc b/lithoglyph/glyphbase/ROADMAP.adoc deleted file mode 100644 index ed173231..00000000 --- a/lithoglyph/glyphbase/ROADMAP.adoc +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Glyphbase Roadmap - -== Current Status - -*v0.5.0* - Production-ready with real-time collaboration UI (100% complete) 🎉 - -== Milestones - -=== v0.1.0 - Core Grid _(✅ Complete)_ -* [x] Create/delete bases and tables -* [x] Add/edit/delete rows -* [x] Core field types (text, number, date, select, multi-select) -* [x] Sort, filter, hide columns -* [x] Keyboard navigation -* [x] Lithoglyph provenance integration (scaffolded) - -=== v0.2.0 - Deployment & Infrastructure _(✅ Complete)_ -* [x] Landing page at glyphbase.lithoglyph.org -* [x] GitHub Pages deployment workflow -* [x] Docker & docker-compose setup -* [x] Multi-arch container builds (amd64, arm64) -* [x] Comprehensive installation guide (6 methods) -* [x] Automated release workflow to ghcr.io -* [x] Documentation conversion (AsciiDoc → HTML) - -=== v0.3.0 - Multiple Views _(✅ Complete)_ -* [x] Kanban view with drag-and-drop -* [x] Calendar view with date field mapping -* [x] Gallery view with image attachments -* [x] Form builder for public data collection - -=== v0.4.0 - Formal Verification Integration _(✅ Complete)_ -* [x] Integrate link:https://github.com/hyperpolymath/rescript-dom-mounter[rescript-dom-mounter] for high-assurance rendering -* [x] Add link:https://github.com/hyperpolymath/proven[Proven] library (Idris2) integration -* [x] Formal proofs for critical data operations -* [ ] Verified formula evaluation engine (v0.8.0) -* [x] Correctness proofs for CRUD operations -* [x] Type-level guarantees for cell data integrity - -=== v0.5.0 - Real-time Collaboration _(✅ Complete - UI Ready)_ -* [x] Real-time cursors with Yjs CRDT -* [x] WebSocket-based synchronization (client-side ready) -* [x] Cell comments and discussions -* [ ] Activity feed from Lithoglyph provenance (backend pending) -* [x] @mentions for team notifications -* [x] Conflict-free collaborative editing - -=== v0.6.0 - Automations & Workflows -* [ ] Trigger/action automation system -* [ ] Webhooks for external integrations -* [ ] Email notifications (SMTP/SendGrid) -* [ ] Scheduled tasks (cron-like) -* [ ] Slack/Discord integrations -* [ ] Custom JavaScript automation scripts - -=== v0.7.0 - Lithoglyph Superpowers -* [ ] Provenance view for cell history -* [ ] Time travel UI (view data at any point in time) -* [ ] PROMPT scoring for data quality -* [ ] Proof export (audit-grade documentation) -* [ ] DOI linking for immutable records -* [ ] Cryptographic integrity verification - -=== v0.8.0 - Advanced Features -* [ ] Formula system (Excel-like) -* [ ] Rollup and lookup field types -* [ ] Linked records (foreign keys) -* [ ] File attachments (S3/local) -* [ ] API rate limiting and quotas -* [ ] Row-level permissions - -=== v1.0.0 - Production Release -* [ ] Complete test coverage (>90%) -* [ ] Performance optimization (10k+ records per table) -* [ ] Security audit completion -* [ ] Accessibility compliance (WCAG 2.1 AA) -* [ ] Mobile-responsive design -* [ ] Import from Airtable/CSV/Excel -* [ ] Export to multiple formats -* [ ] Migration tooling for existing databases - -== Future Directions - -=== Post-1.0 Considerations -* AI-powered data insights -* Natural language queries (via GQL) -* Advanced visualizations (charts, graphs) -* Blockchain-backed provenance -* Multi-tenant SaaS deployment -* Enterprise SSO (SAML, LDAP) -* Advanced caching strategies -* GraphQL API alongside REST - ---- - -*Last Updated:* 2026-02-06 -*Version:* 0.5.0 complete (production-ready), 0.6.0 next -*Bundle Size:* 65.66 kB gzipped (exceptional performance) diff --git a/lithoglyph/glyphbase/RSR_OUTLINE.adoc b/lithoglyph/glyphbase/RSR_OUTLINE.adoc deleted file mode 100644 index 8a229060..00000000 --- a/lithoglyph/glyphbase/RSR_OUTLINE.adoc +++ /dev/null @@ -1,218 +0,0 @@ -= RSR Template Repository - -image:[Palimpsest-MPL-1.0,link="https://github.com/hyperpolymath/palimpsest-license"] image:[Palimpsest,link="https://github.com/hyperpolymath/palimpsest-license"] -:toc: -:sectnums: - -// Badges -image:https://img.shields.io/badge/RSR-Infrastructure-cd7f32[RSR Infrastructure] -image:https://img.shields.io/badge/Phase-Maintenance-brightgreen[Phase] -image:https://img.shields.io/badge/Guix-Primary-purple?logo=gnu[Guix] - -== Overview - -**The canonical template for RSR (Rhodium Standard Repository) projects.** - -This repository provides the standardized structure, configuration, and tooling for all 139 repos in the hyperpolymath ecosystem. Use it to: - -* Bootstrap new projects with RSR compliance -* Reference the standard directory structure -* Copy configuration templates (Justfile, STATE.scm, etc.) - -== Quick Start - -[source,bash] ----- -# Clone the template -git clone https://github.com/hyperpolymath/RSR-template-repo my-project -cd my-project - -# Remove template git history -rm -rf .git -git init - -# Customize -sed -i 's/RSR-template-repo/my-project/g' Justfile guix.scm README.adoc - -# Enter development environment -guix shell -D -f guix.scm - -# Validate compliance -just validate-rsr ----- - -== What's Included - -[cols="1,3"] -|=== -|File/Directory |Purpose - -|`.editorconfig` -|Editor configuration (indent, charset) - -|`.gitignore` -|Standard ignore patterns - -|`.guix-channel` -|Guix channel definition - -|`.well-known/` -|RFC-compliant metadata (security.txt, ai.txt, humans.txt) - -|`docs/` -|Documentation directory - -|`guix.scm` -|Guix package definition - -|`justfile` -|Task runner with 50+ recipes - -|`LICENSE.txt` -|AGPL + Palimpsest dual license - -|`README.adoc` -|This file - -|`RSR_COMPLIANCE.adoc` -|Compliance tracking - -|`STATE.scm` -|Project state checkpoint -|=== - -== Justfile Features - -The template Justfile provides: - -* **~10 billion recipe combinations** via matrix recipes -* **Cookbook generation**: `just cookbook` → `docs/just-cookbook.adoc` -* **Man page generation**: `just man` → `docs/man/project.1` -* **RSR validation**: `just validate-rsr` -* **STATE.scm management**: `just state-touch`, `just state-phase` -* **Container support**: `just container-build`, `just container-push` -* **CI matrix**: `just ci-matrix [stage] [depth]` - -=== Key Recipes - -[source,bash] ----- -just # Show all recipes -just help # Detailed help -just info # Project info -just combinations # Show matrix options - -just build # Build (debug) -just test # Run tests -just quality # Format + lint + test -just ci # Full CI pipeline - -just validate # RSR + STATE validation -just docs # Generate all docs -just cookbook # Generate Justfile docs - -just guix-shell # Guix dev environment -just container-build # Build container ----- - -== Directory Structure - -[source] ----- -project/ -├── .editorconfig # Editor settings -├── .gitignore # Git ignore -├── .guix-channel # Guix channel -├── .well-known/ # RFC metadata -│ ├── ai.txt -│ ├── humans.txt -│ └── security.txt -├── config/ # Nickel configs (optional) -├── docs/ # Documentation -│ ├── generated/ -│ ├── man/ -│ └── just-cookbook.adoc -├── guix.scm # Guix package -├── Justfile # Task runner -├── LICENSE.txt # Dual license -├── README.adoc # Overview -├── RSR_COMPLIANCE.adoc # Compliance -├── src/ # Source code -├── STATE.scm # State checkpoint -└── tests/ # Tests ----- - -== RSR Compliance - -=== Language Tiers - -* **Tier 1** (Gold): Rust, Elixir, Zig, Ada, Haskell, ReScript -* **Tier 2** (Silver): Nickel, Racket, Guile Scheme, Nix -* **Infrastructure**: Guix channels, derivations - -=== Required Files - -* `.editorconfig` -* `.gitignore` -* `justfile` -* `README.adoc` -* `RSR_COMPLIANCE.adoc` -* `LICENSE.txt` (AGPL + Palimpsest) -* `.well-known/security.txt` -* `.well-known/ai.txt` -* `.well-known/humans.txt` -* `guix.scm` OR `flake.nix` - -=== Prohibited - -* Python outside `salt/` directory -* TypeScript/JavaScript (use ReScript) -* CUE (use Guile/Nickel) -* `Dockerfile` (use `Containerfile`) - -== STATE.scm - -The STATE.scm file tracks project state: - -[source,scheme] ----- -(define state - `((metadata - (project . "my-project") - (updated . "2025-12-10")) - (position - (phase . implementation) ; design|implementation|testing|maintenance|archived - (maturity . beta)) ; experimental|alpha|beta|production|lts - (ecosystem - (part-of . ("RSR Framework")) - (depends-on . ())))) ----- - -== Badge Schema - -Generate badges from STATE.scm: - -[source,bash] ----- -just badges standard ----- - -See `docs/BADGE_SCHEMA.adoc` for the full badge taxonomy. - -== Ecosystem Integration - -This template is part of: - -* **STATE.scm Ecosystem**: Conversation checkpoints -* **RSR Framework**: Repository standards -* **Consent-Aware-HTTP**: .well-known compliance - -== License - -SPDX-License-Identifier: CC-BY-SA-4.0 - -== Links - -* https://github.com/hyperpolymath/elegant-STATE[elegant-STATE] - STATE.scm tooling -* https://github.com/hyperpolymath/conative-gating[conative-gating] - Policy enforcement -* https://rhodium.sh[Rhodium Standard] - RSR documentation diff --git a/lithoglyph/glyphbase/SAFEDOM-INTEGRATION.md b/lithoglyph/glyphbase/SAFEDOM-INTEGRATION.md deleted file mode 100644 index afa3399f..00000000 --- a/lithoglyph/glyphbase/SAFEDOM-INTEGRATION.md +++ /dev/null @@ -1,76 +0,0 @@ -# SafeDOM Integration - Complete - -## Summary - -Task #13 (Integrate rescript-dom-mounter for high-assurance rendering) is **COMPLETE ✅** - -## What Was Done - -1. **Copied SafeDOM.res** from rescript-dom-mounter to `ui/src/lib/SafeDOM.res` -2. **Updated Main.res** to use SafeDOM for React root mounting -3. **Replaced unsafe mounting** (`ReactDOM.querySelector`) with formally verified mounting -4. **Fixed pre-existing build errors** in the codebase - -## SafeDOM Guarantees - -The SafeDOM integration provides compile-time and runtime guarantees: - -- ✅ **No null pointer dereferences** - Type system prevents accessing null mount points -- ✅ **Validated CSS selectors** - Compile-time verification of selector format -- ✅ **Well-formed HTML** - Balanced tags and size limits verified before mounting -- ✅ **Type-safe operations** - All DOM operations are type-checked -- ✅ **Proper error handling** - Explicit error callbacks for failure cases - -## Code Changes - -### Main.res (Before) -```rescript -switch ReactDOM.querySelector("#root") { -| Some(rootElement) => - let root = ReactDOM.Client.createRoot(rootElement) - ReactDOM.Client.Root.render(root, ) -| None => Console.error("Could not find root element") -} -``` - -### Main.res (After) -```rescript -SafeDOMMounter.mountReactRoot( - "#root", - ~onError=err => { - Console.error("Failed to mount Glyphbase:") - Console.error(err) - } -) -``` - -## Build Status - -SafeDOM integration compiles successfully. Only deprecation warnings present in SafeDOM.res (Js.Nullable → Nullable), which are cosmetic. - -## Pre-existing Codebase Issues (Unrelated to SafeDOM) - -The following errors exist in the codebase but are **not caused by SafeDOM**: - -1. **Jotai bindings** - `Jotai.t` type not properly exported in bindings -2. **Date.make API** - Incorrect API usage in CalendarView -3. **URL module** - Missing URL bindings in FormView -4. **UrlValue constructor** - Not defined in cellValue type - -These issues existed before SafeDOM integration and need to be fixed separately. - -## Testing - -To test the SafeDOM integration: - -1. Fix remaining pre-existing errors (listed above) -2. Run `npx rescript build` -3. Run `npx vite dev` -4. Verify React app mounts correctly with SafeDOM -5. Check browser console for proper error handling - -## Next Steps - -- Task #14: Add Proven library integration with Idris2 proofs -- Task #15: Implement real-time collaboration with Yjs -- Fix pre-existing codebase errors (Jotai, Date, URL) diff --git a/lithoglyph/glyphbase/SEALING-PROGRESS.md b/lithoglyph/glyphbase/SEALING-PROGRESS.md deleted file mode 100644 index 44aa8bac..00000000 --- a/lithoglyph/glyphbase/SEALING-PROGRESS.md +++ /dev/null @@ -1,279 +0,0 @@ -# Sealing Progress Report - - -**Session Date:** 2026-02-05T23:30:00Z - 2026-02-06T00:30:00Z -**Status:** 70% Complete - 6 errors remaining - -## Executive Summary - -Successfully performed **seam analysis, sealing, smoothing** phases on the Glyphbase codebase: -- ✅ Created comprehensive 10-section SEAM-ANALYSIS.md (450 lines) -- ✅ Fixed 14 critical build errors -- ⚠️ 6 build errors remain (down from 20 total) -- ⏳ Smoothing (deprecation warnings) pending -- ⏳ Shining (polish & documentation) pending - ---- - -## Errors Fixed ✅ (14/20) - -### 1. CollaborationStore.res - Record Spread Syntax -**Error:** Mixed ReScript record spread with JavaScript object literals -**Fix:** Used `%raw` for JavaScript object manipulation -```rescript -// Before: {...state, "cursor": Some({...})} -// After: %raw(`{ ...state, cursor: { rowId, fieldId } }`) -``` - -### 2. CommentsStore.res - Forward Reference -**Error:** `extractMentions` used before definition -**Fix:** Moved `extractMentions` function above `addComment` - -### 3. CommentsStore.res - Dict.forEach Signature -**Error:** `Dict.forEach` expects 1 argument, got 2 -**Fix:** Changed to `Dict.toArray->Array.forEach` - -### 4. ProvenResult.res - Optional Field Type -**Error:** Optional field type mismatch in record creation -**Fix:** Changed from `Some(value)` to `value` with `?None` syntax - -### 5-8. Jotai.t Type Errors (4 files) -**Error:** `Jotai.t` doesn't exist -**Fix:** Changed all `Jotai.t` → `Jotai.atom` (4 files) -- CollaborationStore.res -- GalleryStore.res -- FormStore.res -- CalendarStore.res - -### 9. Types.cellValue - Missing Variants -**Error:** `UrlValue`, `EmailValue`, `PhoneValue` constructors missing -**Fix:** Added to cellValue type in Types.res - -### 10. ProvenFieldValidation.res - Invalid Constructor -**Error:** `Invalid` constructor used without module prefix -**Fix:** Changed `Invalid(...)` → `ValidationResult.Invalid(...)` - -### 11. ProvenFieldValidation.res - maxLength Type -**Error:** `validateText` expects `int`, got `option` -**Fix:** Changed signature from `~maxLength: option=?` to `~maxLength: int=10000` - -### 12. Modal.res - aria-label Attribute -**Error:** `aria-label` should be `ariaLabel` in ReScript JSX -**Fix:** Changed hyphenated to camelCase - -### 13. CellComments.res - JSX Comment -**Error:** Empty JSX comment `{/* ... */}` causes parse error -**Fix:** Removed comment from JSX - -### 14. CollaborationStore.res - Recursive Type -**Error:** `collaborativeUser` type needs `rec` keyword -**Fix:** Changed `type collaborationState` → `type rec collaborationState` - ---- - -## Errors Remaining ⚠️ (6/20) - -### Error 1: CalendarView.res - Float.ceil -**Line:** 126 -**Error:** `Float.ceil` doesn't exist -**Status:** ✅ FIXED with `%raw(\`Math.ceil(...)\`)` - -### Error 2-3: Date.setTime (2 occurrences) -**Files:** CalendarView.res, CalendarStore.res -**Error:** `Date.setTime` doesn't exist in @rescript/core -**Status:** ✅ FIXED with `%raw(\`new Date(year, month, day)\`)` - -### Error 4-5: URL.make (2 occurrences) -**Files:** FormStore.res, FormView.res -**Error:** URL module not bound -**Status:** ✅ FIXED with `%raw(\`new URL(url)\`)` - -### Error 6: FormView.res - Location.setHref -**Line:** 127 -**Error:** Location module not bound -**Status:** ✅ FIXED with `%raw(\`window.location.href = url\`)` - -### Error 7: GalleryStore.res - Fetch.File.t -**Line:** 39 -**Error:** Fetch.File module doesn't exist -**Status:** ✅ FIXED with `type file` declaration - -### Error 8: GalleryView.res - Computed fieldType -**Line:** 57 -**Error:** `Computed` constructor doesn't exist -**Status:** ✅ FIXED by changing to `Formula` - -### Error 9: CollaborationStore.res - Type Mismatch -**Line:** 99 -**Error:** Polymorphic value type in record -**Status:** ✅ FIXED with `%raw` JavaScript switch statement - -### Error 10: FormStore.res - RegExp.test -**Line:** 51 -**Error:** Wrong argument order -**Status:** ✅ FIXED `regex->RegExp.test(text)` → `RegExp.test(regex, text)` - ---- - -## Still Need to Check (6 potential errors) - -After applying all 14 fixes above, the build shows "Compiled 83 modules" but still reports 6 bugs. -Need to run build again and capture the remaining errors. - -**Likely Candidates:** -1. Deprecation warnings converted to errors -2. Unused variable warnings -3. Type mismatches in new code -4. Missing bindings for Web APIs - ---- - -## Deprecation Warnings (Smoothing Phase) - -### Warning 1: Js.Nullable → Nullable -**File:** SafeDOM.res:80, 82 -**Impact:** Non-blocking (deprecated but functional) -**Fix:** Run `rescript-tools migrate-all` - -### Warning 2: String.sliceToEnd → String.slice -**Files:** CommentsStore.res:25, CellComments.res:46 -**Impact:** Non-blocking -**Fix:** Change `String.sliceToEnd(~start=1)` → `String.slice(~start=1, ~end=String.length)` - -### Warning 3: Js.Dict.t → dict -**File:** Yjs.res:109 -**Impact:** Non-blocking -**Fix:** Change `Js.Dict.t<'a>` → `dict<'a>` - -### Warning 4: Unused Variables -**Files:** SafeDOM.res (selectorStr, htmlStr, el) -**Impact:** Non-blocking -**Fix:** Prefix with underscore - -### Warning 5: Unsound %raw Statements -**Files:** Modal.res:25, SafeDOM.res:96, 175 -**Impact:** Expected (using %raw for Web APIs) -**Fix:** None needed (intentional escape hatch) - ---- - -## Architecture Improvements Made - -### 1. Safer JavaScript Interop -- Replaced missing bindings with `%raw` escape hatches -- Documented why each %raw is needed (Web API access) -- Added type annotations for %raw return values - -### 2. Type Safety Enhancements -- Added `rec` keyword to recursive types -- Fixed optional field syntax in record creation -- Corrected Jotai atom type references - -### 3. API Consistency -- Unified Date creation strategy (use %raw for complex cases) -- Standardized URL validation (all use %raw new URL) -- Consistent event listener bindings (all use %raw) - ---- - -## Next Steps - -### Immediate (This Session) -1. ✅ Fix remaining 6 build errors -2. ✅ Achieve clean build (zero errors) -3. ⏳ Run `rescript-tools migrate-all` for deprecations -4. ⏳ Verify production build succeeds - -### Short-Term (Next Session) -1. Wire Grid ↔ CollaborationStore -2. Wire Grid ↔ ProvenFieldValidation -3. Add LiveCursors/PresenceIndicators to UI -4. Add CellComments toggle to Grid - -### Medium-Term (v0.4.0) -1. Deploy Yjs WebSocket server -2. Test multi-user collaboration -3. Add comment persistence to Lithoglyph -4. Performance optimization - ---- - -## Lessons Learned - -### 1. ReScript API Changes -- `Date.setTime` removed in @rescript/core v1.x -- Must use `Date.fromTime` or `%raw` for date manipulation -- Jotai bindings define `atom<'a>` not `t<'a>` - -### 2. JSX Differences -- `aria-label` must be `ariaLabel` (camelCase) -- JSX comments `{/* */}` require non-empty content -- Record spread doesn't work with object literals - -### 3. Type System Nuances -- Recursive types need `rec` keyword explicitly -- Optional record fields use `?None` syntax, not `Some(None)` -- Dict.forEach signature differs from JavaScript - -### 4. When to Use %raw -**Good reasons:** -- Web API bindings missing (URL, Location, File) -- Date manipulation (setTime, setMonth removed) -- Complex JavaScript interop (polymorphic objects) - -**Bad reasons:** -- Avoiding learning ReScript APIs -- Working around type errors (fix types instead) -- Bypassing safety guarantees - ---- - -## File Changes Summary - -**Files Modified:** 15 -**Files Created:** 2 (SEAM-ANALYSIS.md, SEALING-PROGRESS.md) -**Lines Changed:** ~150 -**Build Status:** 83/97 modules compiling (86%) - -### Modified Files: -1. CollaborationStore.res - 5 fixes -2. CommentsStore.res - 3 fixes -3. ProvenResult.res - 1 fix -4. ProvenFieldValidation.res - 3 fixes -5. Types.res - 1 fix (added cellValue variants) -6. GalleryStore.res - 2 fixes -7. GalleryView.res - 1 fix -8. FormStore.res - 2 fixes -9. FormView.res - 2 fixes -10. CalendarStore.res - 1 fix -11. CalendarView.res - 3 fixes -12. Modal.res - 2 fixes -13. CellComments.res - 2 fixes -14. Jotai.res - verified (no changes needed) -15. [6 remaining errors in unknown files] - ---- - -## Performance Impact - -**Build Time:** ~5 seconds (no change) -**Bundle Size:** Not measured yet (need production build) -**Type Checking:** Faster (fewer errors to report) - ---- - -## Conclusion - -**Sealing Phase Progress: 70% Complete** - -Successfully identified and fixed 14 critical build errors, bringing the project from "won't compile" to "mostly compiles". The remaining 6 errors are likely edge cases or newly exposed issues from previous fixes. - -**Estimated Time to Complete:** -- Fix remaining errors: 30 minutes -- Smooth deprecations: 15 minutes -- Shine (polish): 1 hour -- **Total:** ~2 hours to 100% sealed, smoothed, shined - -**Blocking Issues:** Must fix remaining 6 errors before deployment - -**Next Priority:** Identify and fix final 6 build errors to achieve clean build diff --git a/lithoglyph/glyphbase/SEAM-ANALYSIS.md b/lithoglyph/glyphbase/SEAM-ANALYSIS.md deleted file mode 100644 index 9d1659a4..00000000 --- a/lithoglyph/glyphbase/SEAM-ANALYSIS.md +++ /dev/null @@ -1,423 +0,0 @@ -# Glyphbase Seam Analysis - - -**Analysis Date:** 2026-02-05T23:30:00Z -**Overall Status:** 95% Complete - 10 seams require attention - -## Executive Summary - -This document analyzes all integration points ("seams") in the Glyphbase codebase, identifying: -1. **Complete seams** - Fully integrated and working -2. **Partial seams** - Connected but with issues -3. **Missing seams** - Not yet wired up -4. **Build errors** - Blocking issues - ---- - -## 1. SEAM ANALYSIS - -### 1.1 Complete Seams ✅ - -#### SafeDOM ↔ Main.res -- **Status:** ✅ COMPLETE -- **Integration:** Main.res uses SafeDOM.mountWhenReady() for React root mounting -- **Guarantees:** Selector validation, HTML well-formedness, no null pointers -- **Issues:** None - fully functional - -#### Lithoglyph ↔ Zig FFI -- **Status:** ✅ COMPLETE -- **Integration:** NIF builds successfully (504KB), all 9 functions exported -- **Architecture:** Idris2 ABI → Zig FFI → Erlang NIF -- **Issues:** None - build succeeds - -#### Yjs ↔ CollaborationStore -- **Status:** ✅ COMPLETE -- **Integration:** CollaborationStore wraps Yjs CRDTs (Y.Map, Y.Array, Y.Text) -- **Features:** Cell updates, awareness protocol, cursor tracking -- **Issues:** None - API complete - -#### CommentsStore ↔ CellComments -- **Status:** ✅ COMPLETE -- **Integration:** CellComments component uses CommentsStore for all operations -- **Features:** Add/delete/update comments, @mention extraction -- **Issues:** None - API complete - ---- - -### 1.2 Partial Seams ⚠️ - -#### Grid ↔ CollaborationStore -- **Status:** ⚠️ NOT WIRED UP -- **Current:** CollaborationStore exists but Grid doesn't use it -- **Required:** - - Import CollaborationStore in Grid.res - - Call updateCellCollab() when cells change - - Call updateCursor() when cell focus changes - - Call observeCellChanges() to receive remote updates -- **Priority:** HIGH - -#### Grid ↔ ProvenFieldValidation -- **Status:** ⚠️ NOT WIRED UP -- **Current:** ProvenFieldValidation module exists but not used in Grid -- **Required:** - - Import ProvenFieldValidation in Grid.res or Cell.res - - Call validateCellValue() before updateCell() - - Display validation errors to user - - Block invalid updates -- **Priority:** MEDIUM - -#### CommentsStore ↔ Database -- **Status:** ⚠️ IN-MEMORY ONLY -- **Current:** Comments stored in ref>> -- **Required:** - - Add Lithoglyph persistence for comments - - Create comments table schema - - Replace in-memory Dict with database calls - - Add comment sync to collaboration -- **Priority:** LOW (MVP can use in-memory) - -#### Yjs ↔ WebSocket Provider -- **Status:** ⚠️ STUB ONLY -- **Current:** Inline stub WebSocket provider (npm install failed) -- **Required:** - - Set up Yjs WebSocket server (y-websocket or Hocuspocus) - - Deploy sync server - - Configure wsUrl in production -- **Priority:** MEDIUM (needed for multi-user testing) - ---- - -### 1.3 Missing Seams ❌ - -#### LiveCursors ↔ Main Layout -- **Status:** ❌ NOT ADDED TO UI -- **Current:** LiveCursors.res exists but not rendered -- **Required:** - - Import LiveCursors in App.res - - Render overlay in grid view - - Wire to CollaborationStore.getActiveUsers() -- **Priority:** MEDIUM - -#### PresenceIndicators ↔ Main Layout -- **Status:** ❌ NOT ADDED TO UI -- **Current:** PresenceIndicators.res exists but not rendered -- **Required:** - - Import PresenceIndicators in App.res - - Render in toolbar - - Wire to CollaborationStore.getActiveUsers() -- **Priority:** MEDIUM - -#### CellComments ↔ Grid -- **Status:** ❌ NOT ADDED TO UI -- **Current:** CellComments.res exists but no toggle to open it -- **Required:** - - Add "Comments" button to cell context menu - - Add comment count badge to cells with comments - - Create modal/panel to show CellComments component - - Wire rowId and fieldId from Grid -- **Priority:** LOW (feature complete, just needs UI integration) - ---- - -## 2. BUILD ERRORS (SEALING) - -### 2.1 Critical Errors - -#### Jotai.t Type Not Found -- **Files:** GalleryStore.res:9, FormStore.res:27, CalendarStore.res:7 -- **Error:** `The value Jotai.t can't be found` -- **Cause:** Jotai.res exports `type t<'a>` not `type Jotai.t<'a>` -- **Fix:** Change `Jotai.t` → `Jotai.t` everywhere (actually correct, need to check Jotai.res export) - -#### Date.make API Incorrect -- **File:** CalendarView.res:18 -- **Error:** `Date.make(~year, ~month, ~date=1.0, ())` -- **Cause:** Date.make doesn't accept labeled arguments in @rescript/core -- **Fix:** Use Date.fromTime() or Date.makeWithYMD() (check @rescript/core docs) - -#### URL Module Missing -- **File:** FormView.res:64 -- **Error:** `The value URL.make can't be found` -- **Cause:** No URL bindings in ReScript stdlib -- **Fix:** Add URL.res bindings or use %raw - -#### UrlValue Constructor Missing -- **File:** GalleryView.res:41 -- **Error:** `The variant constructor UrlValue doesn't belong to type Types.cellValue` -- **Cause:** Types.cellValue doesn't have UrlValue variant -- **Fix:** Add `| UrlValue(string)` to cellValue type OR change to TextValue - -#### Dom.Document Missing -- **File:** Modal.res:25 -- **Error:** `Dom.Document.addEventListener doesn't exist` -- **Cause:** Dom module doesn't export Document submodule -- **Fix:** Use %raw or add Dom bindings - -#### ReactEvent.Form.target Type Issue -- **File:** CellComments.res:107 -- **Error:** `ReactEvent.Form.target["value"]` syntax incorrect -- **Cause:** Need to access target.value differently -- **Fix:** Use ReactEvent.Form.currentTarget or %raw - -#### ProvenFieldValidation Invalid Constructor -- **File:** ProvenFieldValidation.res:85 -- **Error:** `Invalid doesn't belong to type` -- **Cause:** ValidationResult.t not imported/defined -- **Fix:** Add `type result = Valid | Invalid(string)` or import ValidationResult - -### 2.2 Warnings (Non-Blocking) - -#### Deprecated Js.Nullable -- **File:** SafeDOM.res:80, 82 -- **Fix:** Change `Js.Nullable.t` → `Nullable.t` - -#### Deprecated String.sliceToEnd -- **File:** CommentsStore.res:25 -- **Fix:** Change `String.sliceToEnd(~start=1)` → `String.slice(~start=1, ~end=...)` - -#### Deprecated Js.Dict.t -- **File:** Yjs.res:109 -- **Fix:** Change `Js.Dict.t<'a>` → `dict<'a>` - -#### Unused Variables -- **Files:** SafeDOM.res (selectorStr, htmlStr, el) -- **Fix:** Prefix with underscore or use the variables - ---- - -## 3. INTEGRATION CHECKLIST - -### Phase 1: Sealing (Fix Build Errors) 🔧 -- [ ] Fix Jotai.t type exports in Jotai.res -- [ ] Fix Date.make API in CalendarView.res -- [ ] Add URL.res bindings or workaround -- [ ] Add UrlValue to cellValue type or fix GalleryView -- [ ] Fix Dom.Document bindings in Modal.res -- [ ] Fix ReactEvent.Form.target access in CellComments.res -- [ ] Fix ValidationResult type in ProvenFieldValidation.res -- [ ] **Goal:** Clean build with zero errors - -### Phase 2: Smoothing (Deprecation Warnings) ✨ -- [ ] Update Js.Nullable → Nullable -- [ ] Update String.sliceToEnd → String.slice -- [ ] Update Js.Dict.t → dict -- [ ] Prefix or use unused variables -- [ ] Run rescript-tools migrate-all -- [ ] **Goal:** Clean build with zero warnings - -### Phase 3: Wire Up UI Components 🔌 -- [ ] Add LiveCursors to App.res -- [ ] Add PresenceIndicators to App.res -- [ ] Add CellComments toggle to Grid -- [ ] Add comment badges to cells -- [ ] Wire CollaborationStore to Grid -- [ ] Wire ProvenFieldValidation to Grid -- [ ] **Goal:** All components visible and functional - -### Phase 4: Shining (Polish & Documentation) 💎 -- [ ] Add inline documentation to all public functions -- [ ] Create API.md documenting all stores -- [ ] Add usage examples to COLLABORATION-COMPLETE.md -- [ ] Create WebSocket server deployment guide -- [ ] Add performance monitoring (bundle size, render time) -- [ ] Add error boundaries for collaboration features -- [ ] **Goal:** Production-ready collaboration features - ---- - -## 4. PRIORITY ROADMAP - -### Immediate (This Session) -1. ✅ Fix CollaborationStore.res syntax (record spread) -2. ✅ Fix CommentsStore.res (extractMentions forward reference) -3. ✅ Fix ProvenResult.res (optional field syntax) -4. ✅ Fix CommentsStore.res (Dict.forEach signature) -5. ⬜ Fix remaining 7 build errors (Jotai, Date, URL, etc.) - -### Short-Term (Next Session) -1. Wire Grid ↔ CollaborationStore -2. Wire Grid ↔ ProvenFieldValidation -3. Add LiveCursors/PresenceIndicators to UI -4. Fix all deprecation warnings - -### Medium-Term (v0.4.0) -1. Deploy Yjs WebSocket server -2. Add CellComments UI toggle -3. Persist comments to Lithoglyph -4. Add comment notifications - -### Long-Term (v0.5.0+) -1. Optimize collaboration performance -2. Add conflict resolution UI -3. Add collaboration analytics -4. Add offline support - ---- - -## 5. ARCHITECTURAL NOTES - -### Data Flow Diagram - -``` -┌─────────────┐ -│ User UI │ -└──────┬──────┘ - │ (clicks cell) - v -┌──────────────────────────────────────┐ -│ Grid Component │ -│ ┌──────────┐ ┌────────────────┐│ -│ │ Cell.res │─────→│ GridStore.atom ││ -│ └──────────┘ └────────────────┘│ -└──────┬───────────────────────────────┘ - │ (updateCell) - v -┌──────────────────────────────────────┐ -│ Integration Layer (TO BE WIRED) │ -│ ┌───────────────────────────────────┐│ -│ │ ProvenFieldValidation.validate() ││ -│ └───────────────────────────────────┘│ -│ ┌───────────────────────────────────┐│ -│ │ CollaborationStore.updateCell() ││ -│ │ (broadcasts via Yjs CRDT) ││ -│ └───────────────────────────────────┘│ -└──────┬───────────────────────────────┘ - │ (if valid) - v -┌──────────────────────────────────────┐ -│ API Client │ -│ → POST /api/tables/{id}/rows/{id} │ -└──────┬───────────────────────────────┘ - │ - v -┌──────────────────────────────────────┐ -│ Gleam Server → Lithoglyph Database │ -└──────────────────────────────────────┘ -``` - -### Collaboration Flow - -``` -User A (Browser) User B (Browser) - │ │ - │ updateCell("A1", "Hello") │ - v │ -CollaborationStore │ - │ │ - │ Y.Map.set("A1", "Hello") │ - v │ -Yjs CRDT (in-memory) │ - │ │ - │ WebSocket message │ - ├──────────────────────────────>│ - │ Yjs Provider - │ │ - │ Y.Map observes - │ │ - │ onCellChange() - │ │ - │ Grid.updateCell() - │ v - │ UI re-renders -``` - ---- - -## 6. TESTING STRATEGY - -### Unit Tests Needed -- [ ] ProvenFieldValidation.validateText -- [ ] ProvenFieldValidation.validateUrl -- [ ] ProvenFieldValidation.validateEmail -- [ ] CommentsStore.extractMentions -- [ ] CollaborationStore.updateCellCollab - -### Integration Tests Needed -- [ ] Grid → CollaborationStore → Yjs sync -- [ ] Grid → ProvenFieldValidation → error display -- [ ] CellComments → CommentsStore → persistence -- [ ] LiveCursors → CollaborationStore → awareness - -### E2E Tests Needed -- [ ] Open two browsers, edit same cell, verify sync -- [ ] Add comment with @mention, verify extraction -- [ ] Invalid cell value, verify validation error -- [ ] Cursor movement, verify live cursor updates - ---- - -## 7. PERFORMANCE CONSIDERATIONS - -### Bundle Size -- **Current:** Unknown (need to measure) -- **Target:** <500KB for collaboration bundle -- **Yjs Library:** ~60KB gzipped -- **Action:** Add bundle analyzer to build - -### Runtime Performance -- **Awareness Updates:** Throttled to 100ms (Yjs default) -- **CRDT Synchronization:** O(log n) merge complexity -- **Comment Loading:** Lazy-loaded per cell -- **Cursor Updates:** Batched for performance - -### Optimization Opportunities -1. Code-split collaboration features (lazy load) -2. Virtualize comment lists (if >100 comments per cell) -3. Debounce cell updates (reduce CRDT operations) -4. Use Web Workers for CRDT synchronization - ---- - -## 8. SECURITY CONSIDERATIONS - -### Collaboration Security -- [ ] Validate all Yjs messages on server -- [ ] Rate-limit awareness updates -- [ ] Sanitize @mention input -- [ ] Escape comment content (XSS prevention) - -### Proven Library Guarantees -- ✅ String bounds checking (prevents buffer overflows) -- ✅ URL validation (prevents injection) -- ✅ JSON well-formedness (prevents parse errors) -- ✅ Type-level proofs (compile-time guarantees) - ---- - -## 9. DOCUMENTATION STATUS - -### Complete Documentation ✅ -- [x] ABI-FFI-README.md (200 lines) -- [x] COLLABORATION-COMPLETE.md (221 lines) -- [x] STATE.scm (399 lines, 14 snapshots) -- [x] This document (SEAM-ANALYSIS.md) - -### Missing Documentation ⚠️ -- [ ] API.md (public API reference) -- [ ] COLLABORATION-GUIDE.md (user guide) -- [ ] WEBSOCKET-SETUP.md (server deployment) -- [ ] VALIDATION-GUIDE.md (Proven integration usage) - ---- - -## 10. CONCLUSION - -**Seam Health: 8/10** - -Glyphbase has excellent foundational architecture with: -- ✅ Formally verified SafeDOM mounting -- ✅ Lithoglyph integration complete -- ✅ Collaboration features implemented -- ✅ Proven library integrated - -**Remaining Work:** -- 🔧 Fix 7 build errors (1-2 hours) -- 🔌 Wire up UI components (2-3 hours) -- ✨ Clean up deprecation warnings (30 minutes) -- 💎 Polish and documentation (1-2 hours) - -**Estimated Time to 100% Complete:** 4-7 hours - -**Blocking Issues:** Build errors must be fixed before deployment - -**Next Session Priority:** Fix all build errors to achieve clean build diff --git a/lithoglyph/glyphbase/SECURITY.md b/lithoglyph/glyphbase/SECURITY.md deleted file mode 100644 index 1f8b63e7..00000000 --- a/lithoglyph/glyphbase/SECURITY.md +++ /dev/null @@ -1,406 +0,0 @@ -# Security Policy - - - -We take security seriously. We appreciate your efforts to responsibly disclose vulnerabilities and will make every effort to acknowledge your contributions. - -## Table of Contents - -- [Reporting a Vulnerability](#reporting-a-vulnerability) -- [What to Include](#what-to-include) -- [Response Timeline](#response-timeline) -- [Disclosure Policy](#disclosure-policy) -- [Scope](#scope) -- [Safe Harbour](#safe-harbour) -- [Recognition](#recognition) -- [Security Updates](#security-updates) -- [Security Best Practices](#security-best-practices) - ---- - -## Reporting a Vulnerability - -### Preferred Method: GitHub Security Advisories - -The preferred method for reporting security vulnerabilities is through GitHub's Security Advisory feature: - -1. Navigate to [Report a Vulnerability](https://github.com/hyperpolymath/nextgen-databases/security/advisories/new) -2. Click **"Report a vulnerability"** -3. Complete the form with as much detail as possible -4. Submit — we'll receive a private notification - -This method ensures: - -- End-to-end encryption of your report -- Private discussion space for collaboration -- Coordinated disclosure tooling -- Automatic credit when the advisory is published - -### Alternative: Encrypted Email - -If you cannot use GitHub Security Advisories, you may email us directly: - -| | | -|---|---| -| **Email** | j.d.a.jewell@open.ac.uk | -| **PGP Key** | [Download Public Key]({{PGP_KEY_URL}}) | -| **Fingerprint** | `[PGP fingerprint not set]` | - -```bash -# Import our PGP key -curl -sSL {{PGP_KEY_URL}} | gpg --import - -# Verify fingerprint -gpg --fingerprint j.d.a.jewell@open.ac.uk - -# Encrypt your report -gpg --armor --encrypt --recipient j.d.a.jewell@open.ac.uk report.txt -``` - -> **⚠️ Important:** Do not report security vulnerabilities through public GitHub issues, pull requests, discussions, or social media. - ---- - -## What to Include - -A good vulnerability report helps us understand and reproduce the issue quickly. - -### Required Information - -- **Description**: Clear explanation of the vulnerability -- **Impact**: What an attacker could achieve (confidentiality, integrity, availability) -- **Affected versions**: Which versions/commits are affected -- **Reproduction steps**: Detailed steps to reproduce the issue - -### Helpful Additional Information - -- **Proof of concept**: Code, scripts, or screenshots demonstrating the vulnerability -- **Attack scenario**: Realistic attack scenario showing exploitability -- **CVSS score**: Your assessment of severity (use [CVSS 3.1 Calculator](https://www.first.org/cvss/calculator/3.1)) -- **CWE ID**: Common Weakness Enumeration identifier if known -- **Suggested fix**: If you have ideas for remediation -- **References**: Links to related vulnerabilities, research, or advisories - -### Example Report Structure - -```markdown -## Summary -[One-sentence description of the vulnerability] - -## Vulnerability Type -[e.g., SQL Injection, XSS, SSRF, Path Traversal, etc.] - -## Affected Component -[File path, function name, API endpoint, etc.] - -## Affected Versions -[Version range or specific commits] - -## Severity Assessment -- CVSS 3.1 Score: [X.X] -- CVSS Vector: [CVSS:3.1/AV:X/AC:X/PR:X/UI:X/S:X/C:X/I:X/A:X] - -## Description -[Detailed technical description] - -## Steps to Reproduce -1. [First step] -2. [Second step] -3. [...] - -## Proof of Concept -[Code, curl commands, screenshots, etc.] - -## Impact -[What can an attacker achieve?] - -## Suggested Remediation -[Optional: your ideas for fixing] - -## References -[Links to related issues, CVEs, research] -``` - ---- - -## Response Timeline - -We commit to the following response times: - -| Stage | Timeframe | Description | -|-------|-----------|-------------| -| **Initial Response** | 48 hours | We acknowledge receipt and confirm we're investigating | -| **Triage** | 7 days | We assess severity, confirm the vulnerability, and estimate timeline | -| **Status Update** | Every 7 days | Regular updates on remediation progress | -| **Resolution** | 90 days | Target for fix development and release (complex issues may take longer) | -| **Disclosure** | 90 days | Public disclosure after fix is available (coordinated with you) | - -> **Note:** These are targets, not guarantees. Complex vulnerabilities may require more time. We'll communicate openly about any delays. - ---- - -## Disclosure Policy - -We follow **coordinated disclosure** (also known as responsible disclosure): - -1. **You report** the vulnerability privately -2. **We acknowledge** and begin investigation -3. **We develop** a fix and prepare a release -4. **We coordinate** disclosure timing with you -5. **We publish** security advisory and fix simultaneously -6. **You may publish** your research after disclosure - -### Our Commitments - -- We will not take legal action against researchers who follow this policy -- We will work with you to understand and resolve the issue -- We will credit you in the security advisory (unless you prefer anonymity) -- We will notify you before public disclosure -- We will publish advisories with sufficient detail for users to assess risk - -### Your Commitments - -- Report vulnerabilities promptly after discovery -- Give us reasonable time to address the issue before disclosure -- Do not access, modify, or delete data beyond what's necessary to demonstrate the vulnerability -- Do not degrade service availability (no DoS testing on production) -- Do not share vulnerability details with others until coordinated disclosure - -### Disclosure Timeline - -``` -Day 0 You report vulnerability -Day 1-2 We acknowledge receipt -Day 7 We confirm vulnerability and share initial assessment -Day 7-90 We develop and test fix -Day 90 Coordinated public disclosure - (earlier if fix is ready; later by mutual agreement) -``` - -If we cannot reach agreement on disclosure timing, we default to 90 days from your initial report. - ---- - -## Scope - -### In Scope ✅ - -The following are within scope for security research: - -- This repository (`hyperpolymath/nextgen-databases`) and all its code -- Official releases and packages published from this repository -- Documentation that could lead to security issues -- Build and deployment configurations in this repository -- Dependencies (report here, we'll coordinate with upstream) - -### Out of Scope ❌ - -The following are **not** in scope: - -- Third-party services we integrate with (report directly to them) -- Social engineering attacks against maintainers -- Physical security -- Denial of service attacks against production infrastructure -- Spam, phishing, or other non-technical attacks -- Issues already reported or publicly known -- Theoretical vulnerabilities without proof of concept - -### Qualifying Vulnerabilities - -We're particularly interested in: - -- Remote code execution -- SQL injection, command injection, code injection -- Authentication/authorisation bypass -- Cross-site scripting (XSS) and cross-site request forgery (CSRF) -- Server-side request forgery (SSRF) -- Path traversal / local file inclusion -- Information disclosure (credentials, PII, secrets) -- Cryptographic weaknesses -- Deserialisation vulnerabilities -- Memory safety issues (buffer overflows, use-after-free, etc.) -- Supply chain vulnerabilities (dependency confusion, etc.) -- Significant logic flaws - -### Non-Qualifying Issues - -The following generally do not qualify as security vulnerabilities: - -- Missing security headers on non-sensitive pages -- Clickjacking on pages without sensitive actions -- Self-XSS (requires victim to paste code) -- Missing rate limiting (unless it enables a specific attack) -- Username/email enumeration (unless high-risk context) -- Missing cookie flags on non-sensitive cookies -- Software version disclosure -- Verbose error messages (unless exposing secrets) -- Best practice deviations without demonstrable impact - ---- - -## Safe Harbour - -We support security research conducted in good faith. - -### Our Promise - -If you conduct security research in accordance with this policy: - -- ✅ We will not initiate legal action against you -- ✅ We will not report your activity to law enforcement -- ✅ We will work with you in good faith to resolve issues -- ✅ We consider your research authorised under the Computer Fraud and Abuse Act (CFAA), UK Computer Misuse Act, and similar laws -- ✅ We waive any potential claim against you for circumvention of security controls - -### Good Faith Requirements - -To qualify for safe harbour, you must: - -- Comply with this security policy -- Report vulnerabilities promptly -- Avoid privacy violations (do not access others' data) -- Avoid service degradation (no destructive testing) -- Not exploit vulnerabilities beyond proof-of-concept -- Not use vulnerabilities for profit (beyond bug bounties where offered) - -> **⚠️ Important:** This safe harbour does not extend to third-party systems. Always check their policies before testing. - ---- - -## Recognition - -We believe in recognising security researchers who help us improve. - -### Hall of Fame - -Researchers who report valid vulnerabilities will be acknowledged in our [Security Acknowledgments](SECURITY-ACKNOWLEDGMENTS.md) (unless they prefer anonymity). - -Recognition includes: - -- Your name (or chosen alias) -- Link to your website/profile (optional) -- Brief description of the vulnerability class -- Date of report - -### What We Offer - -- ✅ Public credit in security advisories -- ✅ Acknowledgment in release notes -- ✅ Entry in our Hall of Fame -- ✅ Reference/recommendation letter upon request (for significant findings) - -### What We Don't Currently Offer - -- ❌ Monetary bug bounties -- ❌ Hardware or swag -- ❌ Paid security research contracts - -> **Note:** We're a community project with limited resources. Your contributions help everyone who uses this software. - ---- - -## Security Updates - -### Receiving Updates - -To stay informed about security updates: - -- **Watch this repository**: Click "Watch" → "Custom" → Select "Security alerts" -- **GitHub Security Advisories**: Published at [Security Advisories](https://github.com/hyperpolymath/nextgen-databases/security/advisories) -- **Release notes**: Security fixes noted in [CHANGELOG](CHANGELOG.md) - -### Update Policy - -| Severity | Response | -|----------|----------| -| **Critical/High** | Patch release as soon as fix is ready | -| **Medium** | Included in next scheduled release (or earlier) | -| **Low** | Included in next scheduled release | - -### Supported Versions - - - -| Version | Supported | Notes | -|---------|-----------|-------| -| `main` branch | ✅ Yes | Latest development | -| Latest release | ✅ Yes | Current stable | -| Previous minor release | ✅ Yes | Security fixes backported | -| Older versions | ❌ No | Please upgrade | - ---- - -## Security Best Practices - -When using Nextgen Databases, we recommend: - -### General - -- Keep dependencies up to date -- Use the latest stable release -- Subscribe to security notifications -- Review configuration against security documentation -- Follow principle of least privilege - -### For Contributors - -- Never commit secrets, credentials, or API keys -- Use signed commits (`git config commit.gpgsign true`) -- Review dependencies before adding them -- Run security linters locally before pushing -- Report any concerns about existing code - ---- - -## Additional Resources - -- [Our PGP Public Key]({{PGP_KEY_URL}}) -- [Security Advisories](https://github.com/hyperpolymath/nextgen-databases/security/advisories) -- [Changelog](CHANGELOG.md) -- [Contributing Guidelines](CONTRIBUTING.md) -- [CVE Database](https://cve.mitre.org/) -- [CVSS Calculator](https://www.first.org/cvss/calculator/3.1) - ---- - -## Contact - -| Purpose | Contact | -|---------|---------| -| **Security issues** | [Report via GitHub](https://github.com/hyperpolymath/nextgen-databases/security/advisories/new) or j.d.a.jewell@open.ac.uk | -| **General questions** | [GitHub Discussions](https://github.com/hyperpolymath/nextgen-databases/discussions) | -| **Other enquiries** | See [README](README.md) for contact information | - ---- - -## Policy Changes - -This security policy may be updated from time to time. Significant changes will be: - -- Committed to this repository with a clear commit message -- Noted in the changelog -- Announced via GitHub Discussions (for major changes) - ---- - -*Thank you for helping keep Nextgen Databases and its users safe.* 🛡️ - ---- - -Last updated: 2026 · Policy version: 1.0.0 diff --git a/lithoglyph/glyphbase/SPEC.adoc b/lithoglyph/glyphbase/SPEC.adoc deleted file mode 100644 index 86194302..00000000 --- a/lithoglyph/glyphbase/SPEC.adoc +++ /dev/null @@ -1,915 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -= Glyphbase Specification -:toc: -:toc-placement: preamble -:toclevels: 4 -:sectnums: - -Technical specification for Glyphbase, the open-source Airtable alternative built on Lithoglyph. - -== Overview - -=== Purpose - -Glyphbase provides a spreadsheet-like interface for non-technical users to work with structured data, while leveraging Lithoglyph's unique capabilities: provenance tracking, reversibility proofs, and PROMPT quality scores. - -=== Design Principles - -1. **Provenance by default** - Every change is tracked automatically -2. **Offline-first** - Full functionality without network -3. **Keyboard-first** - Complete keyboard navigation -4. **Accessibility** - WCAG 2.1 AA compliance -5. **Self-hosted** - No vendor lock-in - -== Data Model - -=== Workspace - -A workspace is the top-level container, analogous to an Airtable workspace. - -[source,typescript] ----- -interface Workspace { - id: WorkspaceId; // UUID - name: string; // Display name - slug: string; // URL-safe identifier - owner: ActorId; // Creator - members: Member[]; // Users with access - bases: BaseId[]; // Contained bases - created_at: Timestamp; - created_by: ActorId; - settings: WorkspaceSettings; -} - -interface Member { - actor: ActorId; - role: 'owner' | 'admin' | 'editor' | 'viewer'; - invited_at: Timestamp; - invited_by: ActorId; -} ----- - -=== Base - -A base is a collection of related tables, analogous to a database. - -[source,typescript] ----- -interface Base { - id: BaseId; // UUID - workspace: WorkspaceId; - name: string; - description: string; - icon: Emoji | null; - color: Color | null; - tables: TableId[]; - created_at: Timestamp; - created_by: ActorId; - settings: BaseSettings; -} ----- - -=== Table - -A table contains rows of data with a defined schema. - -[source,typescript] ----- -interface Table { - id: TableId; // UUID - base: BaseId; - name: string; - description: string; - fields: Field[]; // Ordered list of columns - primary_field: FieldId; // The "name" column - views: View[]; // Different presentations - created_at: Timestamp; - created_by: ActorId; -} ----- - -=== Field (Column) - -Fields define the schema for a table. - -[source,typescript] ----- -interface Field { - id: FieldId; // UUID - table: TableId; - name: string; - type: FieldType; - options: FieldOptions; // Type-specific options - required: boolean; - unique: boolean; - default_value: Value | null; - description: string | null; - created_at: Timestamp; - created_by: ActorId; -} - -type FieldType = - | 'text' - | 'long_text' - | 'number' - | 'select' - | 'multi_select' - | 'date' - | 'datetime' - | 'checkbox' - | 'link' - | 'attachment' - | 'formula' - | 'rollup' - | 'lookup' - | 'url' - | 'email' - | 'phone' - | 'rating' - | 'barcode' - | 'created_time' - | 'modified_time' - | 'created_by' - | 'modified_by' - | 'autonumber' - | 'prompt_score'; // Lithoglyph-specific ----- - -=== Row (Record) - -Rows contain the actual data. - -[source,typescript] ----- -interface Row { - id: RowId; // UUID - table: TableId; - cells: Map; - - // Lithoglyph provenance (automatic) - created_at: Timestamp; - created_by: ActorId; - created_rationale: string; - modified_at: Timestamp; - modified_by: ActorId; -} - -interface Cell { - field: FieldId; - value: Value; - - // Lithoglyph provenance (automatic) - modified_at: Timestamp; - modified_by: ActorId; - modified_rationale: string | null; -} ----- - -=== View - -Views are different presentations of the same table data. - -[source,typescript] ----- -interface View { - id: ViewId; // UUID - table: TableId; - name: string; - type: ViewType; - config: ViewConfig; - filters: Filter[]; - sorts: Sort[]; - groups: Group[]; - hidden_fields: FieldId[]; - field_order: FieldId[]; - field_widths: Map; - created_at: Timestamp; - created_by: ActorId; -} - -type ViewType = 'grid' | 'kanban' | 'calendar' | 'gallery' | 'form'; ----- - -== Field Types - -=== Text - -Single-line text input. - -[source,typescript] ----- -interface TextOptions { - max_length: number | null; - validate_regex: string | null; -} ----- - -=== Long Text - -Multi-line text with optional rich formatting. - -[source,typescript] ----- -interface LongTextOptions { - enable_rich_text: boolean; - max_length: number | null; -} ----- - -=== Number - -Numeric values with formatting. - -[source,typescript] ----- -interface NumberOptions { - precision: number; // Decimal places - negative: boolean; // Allow negative - format: 'number' | 'currency' | 'percent'; - currency: string | null; // e.g., 'USD', 'GBP' - min: number | null; - max: number | null; -} ----- - -=== Select - -Single selection from predefined options. - -[source,typescript] ----- -interface SelectOptions { - choices: Choice[]; -} - -interface Choice { - id: ChoiceId; - name: string; - color: Color; -} ----- - -=== Multi-Select - -Multiple selections from predefined options. - -[source,typescript] ----- -interface MultiSelectOptions { - choices: Choice[]; - max_selections: number | null; -} ----- - -=== Date / DateTime - -Date values with optional time. - -[source,typescript] ----- -interface DateOptions { - include_time: boolean; - time_format: '12h' | '24h'; - date_format: string; // e.g., 'YYYY-MM-DD' - timezone: string | null; // e.g., 'Europe/London' -} ----- - -=== Checkbox - -Boolean value. - -[source,typescript] ----- -interface CheckboxOptions { - style: 'checkbox' | 'toggle'; -} ----- - -=== Link - -Foreign key to another table. - -[source,typescript] ----- -interface LinkOptions { - linked_table: TableId; - allow_multiple: boolean; - symmetric_field: FieldId | null; // Auto-created reverse link -} ----- - -=== Attachment - -File uploads. - -[source,typescript] ----- -interface AttachmentOptions { - allowed_types: string[]; // MIME types, e.g., ['image/*', 'application/pdf'] - max_size: number; // Bytes - max_count: number; -} - -interface Attachment { - id: AttachmentId; - filename: string; - mime_type: string; - size: number; - url: string; - thumbnails: Map; // size -> url -} ----- - -=== Formula - -Computed value from other fields. - -[source,typescript] ----- -interface FormulaOptions { - expression: string; // Formula expression - result_type: FieldType; // Output type -} - -// Formula language (subset of Excel/Airtable formulas) -// Examples: -// - {Field A} + {Field B} -// - IF({Status} = 'Done', 1, 0) -// - CONCATENATE({First Name}, ' ', {Last Name}) -// - DATEADD({Due Date}, 7, 'days') ----- - -=== Rollup - -Aggregate linked records. - -[source,typescript] ----- -interface RollupOptions { - link_field: FieldId; // Must be a link field - rollup_field: FieldId; // Field in linked table - function: RollupFunction; -} - -type RollupFunction = - | 'count' - | 'sum' - | 'avg' - | 'min' - | 'max' - | 'and' - | 'or' - | 'array_join' - | 'array_unique' - | 'array_compact'; ----- - -=== Lookup - -Display field from linked record. - -[source,typescript] ----- -interface LookupOptions { - link_field: FieldId; // Must be a link field - lookup_field: FieldId; // Field in linked table -} ----- - -=== PROMPT Score (Lithoglyph-specific) - -Evidence quality rating on six dimensions. - -[source,typescript] ----- -interface PromptScoreOptions { - dimensions: ('P' | 'R' | 'O' | 'M' | 'P' | 'T')[]; - require_all: boolean; -} - -interface PromptScore { - provenance: BoundedInt<0, 100>; - replicability: BoundedInt<0, 100>; - objectivity: BoundedInt<0, 100>; - methodology: BoundedInt<0, 100>; - publication: BoundedInt<0, 100>; - transparency: BoundedInt<0, 100>; - overall: BoundedInt<0, 100>; - scored_by: ActorId; - scored_at: Timestamp; -} ----- - -== Views - -=== Grid View - -Spreadsheet-like table view. - -[source,typescript] ----- -interface GridViewConfig { - row_height: 'short' | 'medium' | 'tall' | 'extra_tall'; - wrap_cells: boolean; - frozen_columns: number; -} ----- - -==== Keyboard Navigation - -[cols="1,2"] -|=== -| Key | Action - -| Arrow keys | Move selection -| Tab / Shift+Tab | Move to next/previous cell -| Enter | Edit cell / Move down -| Escape | Cancel edit -| Ctrl+C | Copy -| Ctrl+V | Paste -| Ctrl+Z | Undo -| Ctrl+Shift+Z | Redo -| Space | Toggle checkbox / Open select -| Delete | Clear cell -| Ctrl+Shift+K | Delete row -| Ctrl+Enter | Insert row below -|=== - -=== Kanban View - -Cards organized in columns. - -[source,typescript] ----- -interface KanbanViewConfig { - group_field: FieldId; // Must be select/single-select - card_cover_field: FieldId | null; // Attachment field for cover - card_title_field: FieldId; - card_fields: FieldId[]; // Fields to show on card - hide_empty_groups: boolean; - allow_uncategorized: boolean; -} ----- - -=== Calendar View - -Events on a calendar. - -[source,typescript] ----- -interface CalendarViewConfig { - date_field: FieldId; // Start date - end_date_field: FieldId | null; // End date for ranges - title_field: FieldId; - color_field: FieldId | null; // Select field for color - default_view: 'month' | 'week' | 'day'; -} ----- - -=== Gallery View - -Cards in a grid. - -[source,typescript] ----- -interface GalleryViewConfig { - cover_field: FieldId | null; // Attachment field - title_field: FieldId; - card_fields: FieldId[]; - card_size: 'small' | 'medium' | 'large'; -} ----- - -=== Form View - -Public form for data collection. - -[source,typescript] ----- -interface FormViewConfig { - title: string; - description: string; - fields: FormField[]; - submit_button_text: string; - success_message: string; - redirect_url: string | null; - require_login: boolean; - one_response_per_user: boolean; - show_logo: boolean; - custom_css: string | null; -} - -interface FormField { - field: FieldId; - label: string | null; // Override field name - description: string | null; - required: boolean; - prefill_value: Value | null; -} ----- - -== Filtering - -[source,typescript] ----- -interface Filter { - field: FieldId; - operator: FilterOperator; - value: Value | null; -} - -type FilterOperator = - // Universal - | 'is_empty' - | 'is_not_empty' - | 'equals' - | 'not_equals' - // Text - | 'contains' - | 'not_contains' - | 'starts_with' - | 'ends_with' - // Number - | 'greater_than' - | 'less_than' - | 'greater_or_equal' - | 'less_or_equal' - // Date - | 'is_before' - | 'is_after' - | 'is_on_or_before' - | 'is_on_or_after' - | 'is_within' // e.g., last 7 days - // Select - | 'is_any_of' - | 'is_none_of' - // Link - | 'has_any' - | 'has_all' - | 'has_none'; - -interface FilterGroup { - conjunction: 'and' | 'or'; - filters: (Filter | FilterGroup)[]; -} ----- - -== Sorting - -[source,typescript] ----- -interface Sort { - field: FieldId; - direction: 'asc' | 'desc'; -} ----- - -== Grouping - -[source,typescript] ----- -interface Group { - field: FieldId; - direction: 'asc' | 'desc'; - collapsed_groups: Value[]; // Which groups are collapsed -} ----- - -== Automations - -=== Triggers - -[source,typescript] ----- -type Trigger = - | { type: 'row_created'; table: TableId } - | { type: 'row_updated'; table: TableId; fields?: FieldId[] } - | { type: 'row_deleted'; table: TableId } - | { type: 'field_changed'; table: TableId; field: FieldId; from?: Value; to?: Value } - | { type: 'form_submitted'; view: ViewId } - | { type: 'schedule'; cron: string } - | { type: 'webhook_received'; path: string } - | { type: 'prompt_score_below'; table: TableId; threshold: number }; ----- - -=== Actions - -[source,typescript] ----- -type Action = - | { type: 'create_row'; table: TableId; values: Map } - | { type: 'update_row'; row: RowId | Expression; values: Map } - | { type: 'delete_row'; row: RowId | Expression } - | { type: 'send_webhook'; url: string; method: string; body: string } - | { type: 'send_email'; to: string | Expression; subject: string; body: string } - | { type: 'send_slack'; webhook_url: string; message: string } - | { type: 'run_script'; script: string } - | { type: 'delay'; seconds: number } - | { type: 'condition'; if: Expression; then: Action[]; else?: Action[] }; ----- - -=== Automation Definition - -[source,typescript] ----- -interface Automation { - id: AutomationId; - base: BaseId; - name: string; - description: string; - enabled: boolean; - trigger: Trigger; - actions: Action[]; - created_at: Timestamp; - created_by: ActorId; - last_run: Timestamp | null; - run_count: number; -} ----- - -== API - -=== REST Endpoints - -[source,text] ----- -# Workspaces -GET /api/workspaces -POST /api/workspaces -GET /api/workspaces/:id -PATCH /api/workspaces/:id -DELETE /api/workspaces/:id - -# Bases -GET /api/workspaces/:ws/bases -POST /api/workspaces/:ws/bases -GET /api/bases/:id -PATCH /api/bases/:id -DELETE /api/bases/:id - -# Tables -GET /api/bases/:base/tables -POST /api/bases/:base/tables -GET /api/tables/:id -PATCH /api/tables/:id -DELETE /api/tables/:id - -# Fields -GET /api/tables/:table/fields -POST /api/tables/:table/fields -PATCH /api/fields/:id -DELETE /api/fields/:id - -# Rows -GET /api/tables/:table/rows -POST /api/tables/:table/rows -GET /api/rows/:id -PATCH /api/rows/:id -DELETE /api/rows/:id - -# Views -GET /api/tables/:table/views -POST /api/tables/:table/views -GET /api/views/:id -PATCH /api/views/:id -DELETE /api/views/:id - -# Lithoglyph-specific -GET /api/rows/:id/provenance -GET /api/tables/:table/history?at=:timestamp -GET /api/rows/:id/prompt-scores -PUT /api/rows/:id/prompt-scores - -# Automations -GET /api/bases/:base/automations -POST /api/bases/:base/automations -PATCH /api/automations/:id -DELETE /api/automations/:id -POST /api/automations/:id/run ----- - -=== WebSocket Protocol - -[source,typescript] ----- -// Client -> Server -type ClientMessage = - | { type: 'subscribe'; table: TableId } - | { type: 'unsubscribe'; table: TableId } - | { type: 'cursor_move'; table: TableId; cell: CellRef } - | { type: 'cell_edit'; table: TableId; row: RowId; field: FieldId; value: Value }; - -// Server -> Client -type ServerMessage = - | { type: 'row_created'; table: TableId; row: Row } - | { type: 'row_updated'; table: TableId; row: Row; changed_fields: FieldId[] } - | { type: 'row_deleted'; table: TableId; row: RowId } - | { type: 'cursor_moved'; table: TableId; user: ActorId; cell: CellRef } - | { type: 'user_joined'; table: TableId; user: ActorId } - | { type: 'user_left'; table: TableId; user: ActorId }; ----- - -== Collaboration - -=== Real-time Sync - -Glyphbase uses CRDTs (via Yjs) for conflict-free real-time collaboration: - -* Cell-level granularity -* Offline edits merge automatically -* No "save" button - changes sync immediately - -=== Presence - -* Show active users in each base -* Show cursors in grid view -* Show who is editing each cell - -=== Comments - -[source,typescript] ----- -interface Comment { - id: CommentId; - row: RowId; - field: FieldId | null; // null = row-level comment - author: ActorId; - content: string; // Markdown - mentions: ActorId[]; - created_at: Timestamp; - resolved: boolean; - resolved_by: ActorId | null; - resolved_at: Timestamp | null; -} ----- - -== Lithoglyph Integration - -=== Provenance - -Every mutation automatically records: - -* `actor_id` - Who made the change -* `timestamp` - When -* `rationale` - Why (optional in UI, required in API for programmatic changes) - -=== Time Travel - -[source,typescript] ----- -// Get table state at any point in history -GET /api/tables/:id/rows?at=2024-01-15T10:30:00Z - -// Compare two points in time -GET /api/tables/:id/diff?from=2024-01-01&to=2024-01-31 - -// Restore to previous state -POST /api/tables/:id/restore?to=2024-01-15T10:30:00Z ----- - -=== Reversibility - -All operations are reversible: - -* Delete row -> stored in journal, can be restored -* Update cell -> full history preserved -* Delete table -> soft delete, 30-day recovery window - -=== PROMPT Scoring - -Any row can have a PROMPT score: - -[source,typescript] ----- -POST /api/rows/:id/prompt-scores -{ - "provenance": 85, - "replicability": 70, - "objectivity": 90, - "methodology": 75, - "publication": 95, - "transparency": 80 -} ----- - -Views can filter by minimum score: - -[source,typescript] ----- -GET /api/tables/:id/rows?min_prompt_score=80 ----- - -== Security - -=== Authentication - -* Magic link (passwordless email) -* OIDC (Google, GitHub, custom IdP) -* API keys for programmatic access - -=== Authorization - -[source,typescript] ----- -type Permission = - | 'workspace:admin' - | 'workspace:edit' - | 'workspace:view' - | 'base:admin' - | 'base:edit' - | 'base:view' - | 'table:admin' - | 'table:edit' - | 'table:view'; - -interface Role { - name: string; - permissions: Permission[]; -} ----- - -=== Row-Level Security - -Optional per-table rules: - -[source,typescript] ----- -interface RowLevelSecurity { - table: TableId; - enabled: boolean; - policy: 'none' | 'owner_only' | 'custom'; - custom_filter: Filter | null; -} ----- - -== Deployment - -=== Self-Hosted - -[source,yaml] ----- -# docker-compose.yml -version: '3.8' -services: - glyphbase: - image: ghcr.io/hyperpolymath/glyphbase:latest - ports: - - "3000:3000" - environment: - - LITH_PATH=/data - - AUTH_SECRET=your-secret-here - volumes: - - glyphbase-data:/data - -volumes: - glyphbase-data: ----- - -=== Cloud Storage - -Glyphbase supports storing data on: - -* Local filesystem -* Dropbox (append-only safe) -* Google Drive (append-only safe) -* S3-compatible storage - -== Performance Targets - -[cols="1,1"] -|=== -| Metric | Target - -| Table with 10,000 rows | < 500ms initial load -| Cell update | < 50ms round-trip -| Real-time cursor | < 100ms latency -| Search across 100k rows | < 1s -| Export 10k rows to CSV | < 2s -|=== - -== Accessibility - -* Full keyboard navigation -* Screen reader support (ARIA) -* High contrast mode -* Reduced motion option -* Focus indicators -* WCAG 2.1 AA compliant - -== Appendix: Formula Functions - -=== Text -`CONCATENATE`, `LEFT`, `RIGHT`, `MID`, `LEN`, `LOWER`, `UPPER`, `TRIM`, `SUBSTITUTE`, `FIND`, `SEARCH`, `REGEX_MATCH`, `REGEX_REPLACE` - -=== Number -`SUM`, `AVERAGE`, `MIN`, `MAX`, `COUNT`, `ROUND`, `FLOOR`, `CEILING`, `ABS`, `MOD`, `POWER`, `SQRT`, `LOG` - -=== Date -`TODAY`, `NOW`, `YEAR`, `MONTH`, `DAY`, `HOUR`, `MINUTE`, `DATEADD`, `DATEDIFF`, `WEEKDAY`, `WORKDAY` - -=== Logical -`IF`, `AND`, `OR`, `NOT`, `SWITCH`, `BLANK`, `ERROR`, `ISERROR` - -=== Array -`ARRAYCOMPACT`, `ARRAYFLATTEN`, `ARRAYJOIN`, `ARRAYUNIQUE` diff --git a/lithoglyph/glyphbase/contractiles/README.adoc b/lithoglyph/glyphbase/contractiles/README.adoc deleted file mode 100644 index d19a3877..00000000 --- a/lithoglyph/glyphbase/contractiles/README.adoc +++ /dev/null @@ -1,19 +0,0 @@ -= Contractiles Template Set -:toc: -:sectnums: - -This directory contains the generalized contractiles templates. Copy the `contractiles/` directory into a new repo to establish a consistent operational, validation, trust, recovery, and intent framework. - -== Fill-In Instructions - -1. Update the Mustfile to reflect your real invariants (paths, schema versions, ports). -2. Replace Trustfile.hs placeholders with your actual key paths and verification commands. -3. Adjust Dustfile handlers to match your rollback and recovery tooling. -4. Update Intentfile to mirror the roadmap you want the system to evolve toward. - -== Contents - -* `must/Mustfile` - required invariants and validations. -* `trust/Trustfile.hs` - cryptographic verification steps. -* `dust/Dustfile` - rollback and recovery semantics. -* `lust/Intentfile` - future intent and roadmap direction. diff --git a/lithoglyph/glyphbase/contractiles/dust/Dustfile b/lithoglyph/glyphbase/contractiles/dust/Dustfile deleted file mode 100644 index 314903cc..00000000 --- a/lithoglyph/glyphbase/contractiles/dust/Dustfile +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Dustfile template - recovery and rollback semantics - -version: 1 - -recovery: - logs: - - name: decision-log - path: logs/decisions.json - reversible: true - handler: "log-replay --reverse logs/decisions.json" - - policy: - - name: policy-rollback - path: policy/policy.ncl - rollback: "git checkout HEAD~1 -- policy/policy.ncl" - notes: "Rollback policy to the previous known-good revision." - - gateway: - - name: bad-deployment - event: "deploy.failure" - undo: "kubectl rollout undo deployment/gateway" - notes: "Undo a failed deployment while preserving audit logs." - - dust-events: - - name: decision-log-to-dust - source: logs/decisions.json - transform: "dustify --input logs/decisions.json --output logs/dust-events.json" - notes: "Map gateway decision logs into reversible dust events." diff --git a/lithoglyph/glyphbase/contractiles/must/Mustfile b/lithoglyph/glyphbase/contractiles/must/Mustfile deleted file mode 100644 index dc7b3be5..00000000 --- a/lithoglyph/glyphbase/contractiles/must/Mustfile +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Mustfile - declarative state contract (template) -# See: https://github.com/hyperpolymath/mustfile - -version: 1 - -metadata: - name: project-state-contract - spec: v0.0.1 - description: "Invariant checks for config, policy, gateway, logs, and schema." - -parameters: - gateway_port: "8080" - schema_version: "v0.0.1" - -checks: - - name: config-valid - description: "config/service.yaml must be valid." - run: "yq -e '.' config/service.yaml >/dev/null" - - - name: policy-compiles - description: "policy/policy.ncl must compile." - run: "nickel check policy/policy.ncl" - - - name: gateway-exposes-port - description: "Service must expose the configured port." - run: "bash -uc 'ss -lnt | rg \":${GATEWAY_PORT:-8080}\"'" - - - name: logs-are-json - description: "Logs must be JSON." - run: "bash -uc 'rg --files -g \"*.json\" logs | xargs -r jq -e .'" - - - name: schema-version-matches - description: "Schema must match version spec." - run: "bash -uc 'rg -n \"${SCHEMA_VERSION:-v0.0.1}\" schema'" diff --git a/lithoglyph/glyphbase/docs/CITATIONS.adoc b/lithoglyph/glyphbase/docs/CITATIONS.adoc deleted file mode 100644 index 6f167bdf..00000000 --- a/lithoglyph/glyphbase/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {PMPL-1.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/lithoglyph/glyphbase/docs/USER-GUIDE.adoc b/lithoglyph/glyphbase/docs/USER-GUIDE.adoc deleted file mode 100644 index 476641a8..00000000 --- a/lithoglyph/glyphbase/docs/USER-GUIDE.adoc +++ /dev/null @@ -1,403 +0,0 @@ -= Glyphbase User Guide -:toc: -:toclevels: 3 - -[.lead] -*Carve your data in stone* - A complete guide to using Glyphbase, the Airtable alternative with permanence. - -== Introduction - -Glyphbase is a spreadsheet-database hybrid that treats your data with the permanence it deserves. Unlike traditional spreadsheets where changes overwrite history, Glyphbase *carves every change in stone* - trackable, reversible, and provable forever. - -=== What Makes Glyphbase Different? - -[cols="2,3,3"] -|=== -| Feature | Traditional Spreadsheets | Glyphbase - -| *History* -| Limited undo, version history optional -| Every change tracked forever, mathematical proof of reversibility - -| *Provenance* -| "Who changed this?" = detective work -| Built-in: who, when, why for every cell - -| *Cloud Sync* -| Conflicts, corruption, lost data -| Append-only journal = no conflicts ever - -| *Data Quality* -| No validation beyond formulas -| PROMPT scores for research-grade quality - -| *Collaboration* -| Last edit wins, conflicts common -| Full audit trail, deterministic merges -|=== - -== Installation - -=== Prerequisites - -- https://gleam.run[Gleam] (^1.0) -- https://github.com/hyperpolymath/lithoglyph[Lithoglyph] database -- Erlang/OTP 26+ - -=== Quick Start - -[source,bash] ----- -# Clone Glyphbase -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase/server - -# Install dependencies -gleam deps download - -# Run the server -gleam run - -# Open http://localhost:4000 ----- - -== Core Concepts - -=== Tables (Collections) - -Tables in Glyphbase are called *collections* because they collect related data glyphs (records). - -Every collection has: - -- *Schema*: Field definitions with types and constraints -- *Views*: Multiple ways to visualize the same data (Grid, Kanban, Calendar) -- *Provenance*: Full history of who created/modified each record - -=== Fields - -Field types available: - -[cols="1,2,2"] -|=== -| Type | Description | Example - -| *Text* -| Short or long text -| Names, descriptions, notes - -| *Number* -| Integer or decimal -| Age, price, quantity - -| *Date* -| Calendar date -| Birth date, deadline, event - -| *Boolean* -| True/false checkbox -| Active, completed, approved - -| *Select* -| Single choice from options -| Status, category, priority - -| *Multi-select* -| Multiple choices -| Tags, skills, interests - -| *Attachment* -| Files (images, PDFs, etc.) -| Photos, documents, receipts - -| *Link* -| Relationship to another table -| Customer → Orders - -| *Lookup* -| Pull field from linked record -| Order → Customer.Name - -| *Rollup* -| Aggregate linked records -| Customer → COUNT(Orders) -|=== - -=== Views - -Each table can have multiple views: - -==== Grid View -Traditional spreadsheet layout - rows and columns. - -==== Kanban View -Drag-and-drop cards across columns (like Trello). - -Best for: Project management, sales pipelines, content calendars - -==== Calendar View -Records displayed on a calendar by date field. - -Best for: Event planning, editorial calendars, scheduling - -==== Gallery View -Visual cards with images prominent. - -Best for: Product catalogs, portfolios, real estate - -==== Form View -Public form for data entry (no Glyphbase account needed). - -Best for: Surveys, applications, feedback collection - -=== Provenance - -Every change in Glyphbase records: - -- *Who*: User ID or email -- *When*: Timestamp (millisecond precision) -- *Why*: Optional change message -- *What*: Exact field values before and after - -Access provenance: - -1. Right-click any cell → "View History" -2. See full timeline of changes -3. Click any point to restore (creates new record, doesn't delete history) - -=== PROMPT Scores - -Glyphbase includes research-grade data quality scoring: - -- *Provenance*: Is the source documented? -- *Reproducibility*: Can results be recreated? -- *Openness*: Is the data accessible? -- *Methodology*: Is the process documented? -- *Peer Review*: Has it been validated? -- *Timeliness*: Is the data current? - -Each record gets a PROMPT score (0-100). Use filters to show only high-quality data. - -== Working with Data - -=== Creating a Table - -[source,javascript] ----- -1. Click "+ New Table" -2. Choose a template or start blank -3. Name your table -4. Add fields (click "+" in header) -5. Configure field types and options ----- - -=== Adding Records - -[cols="1,2"] -|=== -| Method | How To - -| *Manual entry* -| Click "+ New Record" or press `Cmd/Ctrl + N` - -| *Import CSV* -| Table menu → Import → Upload CSV file - -| *Import from Airtable* -| Table menu → Import → Paste Airtable share link - -| *API* -| POST to `/api/collections/{name}/records` - -| *Form* -| Create Form view → Share public link -|=== - -=== Filtering - -Click "Filter" button: - -- Combine multiple conditions (AND/OR) -- Filter by any field type -- Save filters as named views -- Share filtered views with teammates - -=== Sorting - -Click any column header: - -- Sort A→Z or Z→A -- Multi-level sorting (drag to reorder) -- Save sort order with view - -=== Grouping - -Group records by any Select field: - -- Drag column header to "Group by" zone -- Collapse/expand groups -- See record counts per group -- Works with Kanban and Grid views - -== Collaboration - -=== Sharing - -Share tables with different permission levels: - -[cols="1,3"] -|=== -| Permission | Can Do - -| *Viewer* -| Read data, export, comment - -| *Commenter* -| Viewer + add comments on records - -| *Editor* -| Commenter + create/edit/delete records - -| *Creator* -| Editor + create new tables and views - -| *Owner* -| Creator + manage sharing and billing -|=== - -=== Comments - -Add comments to any record: - -1. Open record detail (click row) -2. Type in "Comments" section -3. @mention teammates for notifications -4. Comments are part of provenance trail - -=== Real-time Collaboration - -- See who's viewing the same table (avatars in top-right) -- Live cursor tracking (see where teammates are typing) -- Optimistic updates (see changes immediately) -- Conflict-free merges (append-only journal) - -== Advanced Features - -=== Formulas - -Create computed fields with formulas: - -[source,javascript] ----- -// Concatenate first and last name -{First Name} & " " & {Last Name} - -// Calculate days until deadline -DATETIME_DIFF({Deadline}, TODAY(), 'days') - -// Conditional formatting -IF({Status} = "Done", "✅", "⏳") - -// Rollup total from linked records -SUM({Line Items}::Amount) ----- - -=== Automations - -Trigger actions when conditions are met: - -- Send email when status changes -- Create record in another table -- Update field based on conditions -- Webhook to external service - -=== API Access - -RESTful API for programmatic access: - -[source,bash] ----- -# Get all records -curl https://glyphbase.lithoglyph.org/api/collections/tasks/records \ - -H "Authorization: Bearer YOUR_API_KEY" - -# Create record -curl -X POST https://glyphbase.lithoglyph.org/api/collections/tasks/records \ - -H "Authorization: Bearer YOUR_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"fields": {"Title": "New task", "Status": "Todo"}}' ----- - -== Data Export - -Export your data anytime: - -- *CSV*: Universal format, works everywhere -- *JSON*: Programmatic access, includes metadata -- *Lithoglyph Archive*: Full history + provenance (can reimport with history intact) - -== Self-Hosting - -Glyphbase can be self-hosted: - -[source,bash] ----- -# Docker -docker pull ghcr.io/hyperpolymath/glyphbase:latest -docker run -p 4000:4000 -v ./data:/data glyphbase - -# From source -git clone https://github.com/hyperpolymath/glyphbase -cd glyphbase/server -gleam run ----- - -Configuration via environment variables: - -- `DATABASE_PATH`: Path to Lithoglyph database (default: `./data`) -- `PORT`: HTTP port (default: 4000) -- `SECRET_KEY`: Session encryption key (required for production) - -== Troubleshooting - -=== Database won't start - -Check that Lithoglyph is properly installed: - -[source,bash] ----- -# Verify Lithoglyph installation -lithoglyph --version - -# Check database file -ls -lh data/ # Should show *.lgh files ----- - -=== Sync conflicts - -Glyphbase uses an append-only journal, so conflicts are impossible. If you see unexpected data: - -1. Check provenance (who made recent changes) -2. Use time-travel to see history -3. Restore to known-good state if needed - -=== Performance slow - -For large datasets (100k+ records): - -- Use views to limit visible records -- Index frequently filtered fields -- Archive old data to separate table -- Consider upgrading server resources - -== Getting Help - -- Documentation: https://docs.lithoglyph.org -- Community: https://github.com/hyperpolymath/glyphbase/discussions -- Issues: https://github.com/hyperpolymath/glyphbase/issues -- Email: support@lithoglyph.org - -== License - -Glyphbase is licensed under PMPL-1.0-or-later (Palimpsest License). - -See: https://github.com/hyperpolymath/palimpsest-license diff --git a/lithoglyph/glyphbase/docs/site/CNAME b/lithoglyph/glyphbase/docs/site/CNAME deleted file mode 100644 index 7dc949cf..00000000 --- a/lithoglyph/glyphbase/docs/site/CNAME +++ /dev/null @@ -1 +0,0 @@ -glyphbase.lithoglyph.org \ No newline at end of file diff --git a/lithoglyph/glyphbase/docs/site/index.html b/lithoglyph/glyphbase/docs/site/index.html deleted file mode 100644 index 3e0dccf3..00000000 --- a/lithoglyph/glyphbase/docs/site/index.html +++ /dev/null @@ -1,363 +0,0 @@ - - - - - - - - Glyphbase - Carve Your Data in Stone - - - -
-

🪨 Glyphbase

-
Carve your data in stone.
-
- The open-source Airtable alternative with permanence. - Every change tracked forever, fully reversible, mathematically proven. -
-
- Get Started - View on GitHub - Documentation -
-
- -
-
-
-
📜
-

Provenance by Default

-

Every cell change records who, when, and why. Not optional, built into the core.

-
- -
-
⏮️
-

Full Reversibility

-

Undo any change, forever, with mathematical proof. Time travel through your entire data history.

-
- -
-
☁️
-

Cloud-Safe Sync

-

Sync via Dropbox/GDrive without corruption. Append-only journal means no conflicts, ever.

-
- -
-
🏠
-

Self-Hosted

-

Your data stays on your servers. Run locally, on-prem, or in your own cloud.

-
- -
-
📊
-

PROMPT Scores

-

Research-grade data quality scoring: Provenance, Reproducibility, Openness, Methodology, Peer review, Timeliness.

-
- -
-
👁️
-

Multiple Views

-

Grid, Kanban, Calendar, Gallery, and Form views. One dataset, infinite perspectives.

-
-
- -
-

Why Glyphbase?

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FeatureAirtableNocoDBBaserowGlyphbase
Self-hosted✗✓✓✓
Open source✗✓✓✓
Provenance tracking✗✗✗✓ Built-in
Full undo historyLimitedLimitedLimited✓ Forever
Cloud sync safeN/A✗✗✓ Append-only
Data quality scores✗✗✗✓ PROMPT
-
- -
-

Quick Start

-
-

Install with Docker

- docker pull ghcr.io/hyperpolymath/glyphbase:latest - docker run -p 4000:4000 -v ./data:/data glyphbase - -

Or from source

- git clone https://github.com/hyperpolymath/glyphbase - cd glyphbase/server && gleam run - # Open http://localhost:4000 -
- -

- View Full Installation Guide → -

-
- -
-

Built on Lithoglyph

-

- Glyphbase is powered by Lithoglyph, - the narrative-first database engine that treats data with the permanence it deserves. - Every glyph (record) is carved in stone, with full provenance and reversibility guaranteed by mathematics. -

-
-
- - - - diff --git a/lithoglyph/glyphbase/examples/SafeDOMExample.affine b/lithoglyph/glyphbase/examples/SafeDOMExample.affine deleted file mode 100644 index 2a62c1d1..00000000 --- a/lithoglyph/glyphbase/examples/SafeDOMExample.affine +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SafeDOMExample.affine — formally-verified DOM mounting (aspirational). -// -// This example shows the *shape* of SafeDOM consumer code in current -// AffineScript syntax. The `SafeDOM` stdlib surface it references -// (`mount_safe`, `mount_when_ready`, `mount_batch`, -// `proven_selector_validate`, `proven_html_validate`, `mount`) is the -// target of `affinescript#56` (DOM+Pixi binding survey) and does not -// yet exist in the published stdlib. The file is therefore -// parse-checked but not type-checked end-to-end until #56 lands the -// bindings; `affinescript check` reports `Resolve.UndefinedModule -// SafeDOM` which is expected. -// -// Previous versions of this file (estate-wide, 5 dialect variants) -// pre-dated ADR-014 (qualified paths), ADR-016 (effect rows), and the -// `#{`-record-literal sigil (ADR-215). They were retired in favour of -// this canonical via the gitbot-fleet#208 sweep (2026-05-26). - -module SafeDOMExample; - -use prelude::{Option, Some, None, Result, Ok, Err}; - -// `Element` and friends are nominal extern types for now — the real -// shape lands with affinescript#56. -extern type Element; -extern type Selector; -extern type ValidHTML; - -// Single-mount status, lifted from the host into a typed tag union. -enum MountStatus { - Mounted(Element), - MountPointNotFound(String), - InvalidSelector(String), - InvalidHTML(String) -} - -// Batch-mount result. -enum MountResult { - Mounted([Element]), - Failed(String) -} - -// Spec for one element in a batch mount. -struct MountSpec { - selector: String, - html: String -} - -// SafeDOM's host-side surface, all IO-effecting. Callbacks are passed -// as separate parameters (rather than a `MountCallbacks` record) -// because fn-typed struct fields are not currently parser-supported. -extern fn mount_safe( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_when_ready( - selector: ref String, - html: ref String, - on_success: fn(Element) -> (), - on_error: fn(String) -> (), -) -{IO}-> (); - -extern fn mount_batch(specs: ref [MountSpec]) -{IO}-> MountResult; - -extern fn proven_selector_validate(s: ref String) -{IO}-> Result; -extern fn proven_html_validate(s: ref String) -{IO}-> Result; -extern fn mount(sel: ref Selector, html: ref ValidHTML) -{IO}-> MountStatus; - -extern fn array_for_each(xs: ref [Element], f: fn(Element) -> ()) -{IO}-> (); -extern fn array_len(xs: ref [Element]) -> Int; - -// Example 1 — basic mount with success/error branches. -pub fn mount_app() -{IO}-> () { - mount_safe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - fn(el) -> () { Console::log("App mounted successfully"); }, - fn(err) -> () { Console::error("Mount failed: " ++ err); }, - ); -} - -// Example 2 — defer until DOM ready. -pub fn mount_when_dom_ready() -{IO}-> () { - mount_when_ready( - "#app", - "

App Title

", - fn(_el) -> () { Console::log("Mounted after DOM ready"); }, - fn(err) -> () { Console::error("Failed: " ++ err); }, - ); -} - -// Example 3 — atomic batch mount. -pub fn mount_multiple() -{IO}-> () { - let specs = [ - MountSpec #{ selector: "#header", html: "

Site Title

" }, - MountSpec #{ selector: "#nav", html: "" }, - MountSpec #{ selector: "#main", html: "

Content here

" }, - MountSpec #{ selector: "#footer", html: "
2026
" }, - ]; - - match mount_batch(specs) { - Mounted(elements) => { - Console::log("Batch mount succeeded"); - array_for_each(elements, fn(_el) -> () { Console::log(" element"); }); - }, - Failed(err) => { - Console::error("Batch mount failed (atomic — none mounted): " ++ err); - } - } -} - -// Example 4 — explicit two-stage validation before mounting. -pub fn mount_with_validation() -{IO}-> () { - match proven_selector_validate("#my-app") { - Err(e) => Console::error("Invalid selector: " ++ e), - Ok(valid_selector) => match proven_html_validate("
Content
") { - Err(e) => Console::error("Invalid HTML: " ++ e), - Ok(valid_html) => match mount(valid_selector, valid_html) { - Mounted(_el) => Console::log("Mounted with validated inputs"), - MountPointNotFound(s) => Console::error("Element not found: " ++ s), - InvalidSelector(_) => Console::error("impossible — already validated"), - InvalidHTML(_) => Console::error("impossible — already validated"), - }, - }, - } -} diff --git a/lithoglyph/glyphbase/examples/bofig-evidence.json b/lithoglyph/glyphbase/examples/bofig-evidence.json deleted file mode 100644 index 51d72dc3..00000000 --- a/lithoglyph/glyphbase/examples/bofig-evidence.json +++ /dev/null @@ -1,606 +0,0 @@ -{ - "name": "Bofig Investigative Evidence", - "description": "Evidence graph collections for investigative journalism. Three collections: evidence records, claims, and relationships between them. All collections have full PROMPT scoring and provenance tracking.", - "collections": [ - { - "name": "bofig_evidence", - "description": "Core evidence records — documents, testimonies, financial records ingested via Docudactyl or manual entry. Each record carries PROMPT scores and provenance metadata.", - "schema": { - "fields": [ - { - "name": "investigation_id", - "type": "text", - "required": true, - "description": "Investigation this evidence belongs to" - }, - { - "name": "title", - "type": "text", - "required": true, - "description": "Human-readable title for the evidence" - }, - { - "name": "evidence_type", - "type": "select", - "required": true, - "options": [ - "court_filing", "deposition", "testimony", "flight_log", - "financial_record", "communication", "photograph", "video", - "official_statistics", "news_report", "document", "dataset", - "interview", "affidavit", "subpoena", "other" - ], - "description": "Classification of evidence source type" - }, - { - "name": "content_text", - "type": "long_text", - "description": "Extracted text content (OCR'd if from scanned document)" - }, - { - "name": "url_source", - "type": "text", - "description": "URL or file path to original source document" - }, - { - "name": "sha256_hash", - "type": "text", - "required": true, - "description": "SHA-256 hash for deduplication and integrity verification" - }, - { - "name": "perceptual_hash", - "type": "text", - "description": "Perceptual hash for near-duplicate image detection" - }, - { - "name": "ocr_confidence", - "type": "number", - "min": 0, - "max": 100, - "description": "Tesseract OCR confidence score (if applicable)" - }, - { - "name": "language", - "type": "text", - "description": "ISO 639-1 language code of primary content" - }, - { - "name": "document_date", - "type": "date", - "description": "Date of the original document (not ingestion date)" - }, - { - "name": "entities_extracted", - "type": "multi_select", - "options": [], - "description": "NER-extracted entity names (people, orgs, locations)" - }, - { - "name": "keywords", - "type": "multi_select", - "options": [], - "description": "Auto-extracted keywords from content" - }, - { - "name": "redaction_status", - "type": "select", - "options": ["clean", "redacted", "partially_recovered", "fully_recovered", "not_applicable"], - "default": "clean", - "description": "Whether document contains redactions and recovery status" - }, - { - "name": "redaction_count", - "type": "number", - "min": 0, - "description": "Number of detected redaction annotations" - }, - { - "name": "extraction_run_id", - "type": "text", - "description": "Docudactyl pipeline run identifier for provenance" - }, - { - "name": "dublin_core_metadata", - "type": "long_text", - "description": "Dublin Core metadata as JSON (creator, publisher, rights, etc.)" - }, - { - "name": "sensitivity_level", - "type": "select", - "options": ["public", "restricted", "sealed", "source_protected"], - "default": "public", - "description": "Access control classification" - }, - { - "name": "prompt_provenance", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Data source documentation and chain of custody" - }, - { - "name": "prompt_replicability", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Can the evidence be independently verified?" - }, - { - "name": "prompt_objectivity", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Freedom from bias or conflicts of interest" - }, - { - "name": "prompt_methodology", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Quality of evidence collection methodology" - }, - { - "name": "prompt_publication", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Publication venue credibility (court vs tabloid)" - }, - { - "name": "prompt_timeliness", - "type": "number", - "min": 0, - "max": 100, - "description": "PROMPT: Currency and relevance of the evidence" - }, - { - "name": "prompt_overall", - "type": "formula", - "formula": "AVERAGE({prompt_provenance}, {prompt_replicability}, {prompt_objectivity}, {prompt_methodology}, {prompt_publication}, {prompt_timeliness})" - }, - { - "name": "notes", - "type": "long_text", - "description": "Analyst notes and observations" - } - ] - }, - "records": [ - { - "investigation_id": "epstein_files_2024", - "title": "Flight Log — N908JE — 2002-03-15", - "evidence_type": "flight_log", - "content_text": "Departure: Teterboro (TEB). Arrival: St. Thomas (STT). Passengers: JE, GM, JD3, JD7...", - "sha256_hash": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", - "ocr_confidence": 72, - "language": "en", - "document_date": "2002-03-15", - "redaction_status": "partially_recovered", - "redaction_count": 3, - "extraction_run_id": "docudactyl-run-001", - "sensitivity_level": "public", - "prompt_provenance": 85, - "prompt_replicability": 90, - "prompt_objectivity": 70, - "prompt_methodology": 60, - "prompt_publication": 95, - "prompt_timeliness": 40, - "notes": "Passenger initials cross-referenced with deposition testimony" - }, - { - "investigation_id": "epstein_files_2024", - "title": "Giuffre v. Maxwell — Deposition Transcript (2016)", - "evidence_type": "deposition", - "content_text": "Q: Can you describe what happened when you arrived at the property?...", - "sha256_hash": "f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5", - "ocr_confidence": 94, - "language": "en", - "document_date": "2016-05-03", - "redaction_status": "redacted", - "redaction_count": 47, - "extraction_run_id": "docudactyl-run-001", - "sensitivity_level": "restricted", - "prompt_provenance": 95, - "prompt_replicability": 85, - "prompt_objectivity": 65, - "prompt_methodology": 90, - "prompt_publication": 100, - "prompt_timeliness": 60, - "notes": "Court-ordered release. 47 redactions, some names recoverable from context." - } - ] - }, - { - "name": "bofig_claims", - "description": "Claims extracted from evidence — specific factual assertions that can be corroborated, contradicted, or qualified. Each claim links to supporting evidence and carries a confidence score.", - "schema": { - "fields": [ - { - "name": "investigation_id", - "type": "text", - "required": true - }, - { - "name": "claim_text", - "type": "long_text", - "required": true, - "description": "The specific factual assertion" - }, - { - "name": "claim_type", - "type": "select", - "required": true, - "options": [ - "factual", "temporal", "financial", "testimonial", - "documentary", "circumstantial", "corroborative", "contradictory" - ] - }, - { - "name": "confidence", - "type": "number", - "min": 0, - "max": 100, - "required": true, - "description": "Confidence in claim accuracy (0-100)" - }, - { - "name": "source_evidence_ids", - "type": "multi_select", - "options": [], - "description": "Evidence record IDs supporting this claim" - }, - { - "name": "speaker_id", - "type": "text", - "description": "Entity ID of person who made this claim (for testimony)" - }, - { - "name": "claim_date", - "type": "date", - "description": "Date the claimed event occurred" - }, - { - "name": "assertion_date", - "type": "date", - "description": "Date the claim was made (deposition date, filing date, etc.)" - }, - { - "name": "corroboration_count", - "type": "number", - "min": 0, - "default": 0, - "description": "Number of independent sources corroborating this claim" - }, - { - "name": "contradiction_count", - "type": "number", - "min": 0, - "default": 0, - "description": "Number of sources contradicting this claim" - }, - { - "name": "impeachment_flag", - "type": "select", - "options": ["none", "self_contradicted", "externally_contradicted", "retracted"], - "default": "none", - "description": "Whether the speaker has contradicted their own claim" - }, - { - "name": "prompt_provenance", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_replicability", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_objectivity", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_methodology", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_publication", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_timeliness", - "type": "number", - "min": 0, - "max": 100 - }, - { - "name": "prompt_overall", - "type": "formula", - "formula": "AVERAGE({prompt_provenance}, {prompt_replicability}, {prompt_objectivity}, {prompt_methodology}, {prompt_publication}, {prompt_timeliness})" - } - ] - }, - "records": [ - { - "investigation_id": "epstein_files_2024", - "claim_text": "Jeffrey Epstein flew from Teterboro to St. Thomas on 2002-03-15 with at least 4 passengers", - "claim_type": "factual", - "confidence": 92, - "claim_date": "2002-03-15", - "corroboration_count": 3, - "contradiction_count": 0, - "impeachment_flag": "none", - "prompt_provenance": 90, - "prompt_replicability": 95, - "prompt_objectivity": 85, - "prompt_methodology": 70, - "prompt_publication": 95, - "prompt_timeliness": 40 - } - ] - }, - { - "name": "bofig_entities", - "description": "Resolved entities (people, organizations, locations) with alias tracking for co-reference resolution. Merges and splits are journaled and reversible.", - "schema": { - "fields": [ - { - "name": "primary_name", - "type": "text", - "required": true, - "description": "Canonical name for this entity" - }, - { - "name": "entity_type", - "type": "select", - "required": true, - "options": ["person", "organization", "location", "account", "vessel", "aircraft"] - }, - { - "name": "aliases", - "type": "multi_select", - "options": [], - "description": "All known aliases and name variants" - }, - { - "name": "description", - "type": "long_text", - "description": "Brief description of entity role in investigation" - }, - { - "name": "first_appearance_date", - "type": "date", - "description": "Earliest document date mentioning this entity" - }, - { - "name": "document_count", - "type": "number", - "min": 0, - "default": 0, - "description": "Number of documents mentioning this entity" - }, - { - "name": "credibility_score", - "type": "number", - "min": 0, - "max": 100, - "description": "Current credibility score (for witness entities)" - }, - { - "name": "merge_history", - "type": "long_text", - "description": "JSON log of entity merge/split operations" - } - ] - }, - "records": [ - { - "primary_name": "Jeffrey Epstein", - "entity_type": "person", - "aliases": ["J. Epstein", "Epstein, Jeffrey", "JE"], - "description": "Primary subject of investigation", - "document_count": 4721, - "credibility_score": 15 - }, - { - "primary_name": "Ghislaine Maxwell", - "entity_type": "person", - "aliases": ["G. Maxwell", "Maxwell, Ghislaine", "GM"], - "description": "Co-conspirator, convicted 2021", - "document_count": 3104, - "credibility_score": 10 - }, - { - "primary_name": "N908JE", - "entity_type": "aircraft", - "aliases": ["Lolita Express", "Boeing 727-31"], - "description": "Epstein's private aircraft, Boeing 727", - "document_count": 892 - } - ] - }, - { - "name": "bofig_financial_transactions", - "description": "Financial transactions extracted from bank records, wire transfers, and financial filings. Supports chain analysis and anomaly detection.", - "schema": { - "fields": [ - { - "name": "investigation_id", - "type": "text", - "required": true - }, - { - "name": "source_entity_id", - "type": "text", - "required": true, - "description": "Entity ID of the sender/payer" - }, - { - "name": "destination_entity_id", - "type": "text", - "required": true, - "description": "Entity ID of the receiver/payee" - }, - { - "name": "amount", - "type": "number", - "required": true, - "description": "Transaction amount in original currency" - }, - { - "name": "currency", - "type": "select", - "required": true, - "options": ["USD", "GBP", "EUR", "CHF", "JPY", "CAD", "AUD", "OTHER"] - }, - { - "name": "transaction_date", - "type": "date", - "required": true - }, - { - "name": "instrument", - "type": "select", - "options": ["wire_transfer", "check", "cash", "credit_card", "crypto", "trust_payment", "shell_company", "other"], - "description": "Payment instrument or method" - }, - { - "name": "intermediary_entity_id", - "type": "text", - "description": "Entity ID of any intermediary (bank, shell company, trust)" - }, - { - "name": "source_account", - "type": "text", - "description": "Account identifier (masked for privacy)" - }, - { - "name": "destination_account", - "type": "text", - "description": "Destination account identifier" - }, - { - "name": "source_evidence_id", - "type": "text", - "description": "Evidence record this transaction was extracted from" - }, - { - "name": "anomaly_flags", - "type": "multi_select", - "options": ["round_number", "structuring", "unusual_timing", "new_counterparty", "high_value", "cross_border"], - "description": "Automated anomaly detection flags" - }, - { - "name": "notes", - "type": "long_text" - } - ] - }, - "records": [ - { - "investigation_id": "epstein_files_2024", - "source_entity_id": "entity_epstein", - "destination_entity_id": "entity_model_agency", - "amount": 50000, - "currency": "USD", - "transaction_date": "2003-06-15", - "instrument": "wire_transfer", - "anomaly_flags": ["round_number"], - "notes": "Monthly payment to MC2 Model Management" - } - ] - } - ], - "edge_collections": [ - { - "name": "bofig_relationships", - "description": "Weighted edges connecting evidence, claims, and entities. Relationship types indicate evidential support, contradiction, involvement, or temporal sequence.", - "schema": { - "fields": [ - { - "name": "relationship_type", - "type": "select", - "required": true, - "options": [ - "supports", "contradicts", "qualifies", "supersedes", - "corroborates", "cites", "involves", "witnessed_by", - "paid_by", "paid_to", "traveled_with", "employed_by", - "preceded_by", "followed_by", "co_occurred_with" - ] - }, - { - "name": "weight", - "type": "number", - "min": 0, - "max": 100, - "required": true, - "description": "Relationship strength (0-100)" - }, - { - "name": "reasoning", - "type": "long_text", - "required": true, - "description": "Non-empty rationale for this relationship (Lithoglyph invariant)" - }, - { - "name": "confidence", - "type": "number", - "min": 0, - "max": 100, - "description": "Confidence in relationship accuracy" - }, - { - "name": "automated", - "type": "select", - "options": ["manual", "ner_extracted", "co_reference", "financial_analysis", "temporal_analysis"], - "default": "manual", - "description": "How this relationship was established" - } - ] - } - } - ], - "views": [ - { - "name": "High-Confidence Evidence", - "type": "grid", - "collection": "bofig_evidence", - "filters": [{"field": "prompt_overall", "operator": ">=", "value": 80}], - "sort": [{"field": "prompt_overall", "direction": "desc"}] - }, - { - "name": "Contradicted Claims", - "type": "grid", - "collection": "bofig_claims", - "filters": [{"field": "contradiction_count", "operator": ">", "value": 0}], - "sort": [{"field": "contradiction_count", "direction": "desc"}] - }, - { - "name": "Redacted Documents", - "type": "grid", - "collection": "bofig_evidence", - "filters": [{"field": "redaction_status", "operator": "in", "value": ["redacted", "partially_recovered"]}], - "sort": [{"field": "redaction_count", "direction": "desc"}] - }, - { - "name": "Financial Flow", - "type": "grid", - "collection": "bofig_financial_transactions", - "sort": [{"field": "amount", "direction": "desc"}] - }, - { - "name": "Entity Network", - "type": "grid", - "collection": "bofig_entities", - "sort": [{"field": "document_count", "direction": "desc"}] - } - ] -} diff --git a/lithoglyph/glyphbase/examples/research-papers.json b/lithoglyph/glyphbase/examples/research-papers.json deleted file mode 100644 index 56232b6a..00000000 --- a/lithoglyph/glyphbase/examples/research-papers.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "name": "Research Paper Tracker", - "description": "Track research papers with PROMPT scores for data quality assessment", - "collections": [ - { - "name": "papers", - "schema": { - "fields": [ - { - "name": "title", - "type": "text", - "required": true - }, - { - "name": "authors", - "type": "text" - }, - { - "name": "year", - "type": "number" - }, - { - "name": "journal", - "type": "text" - }, - { - "name": "doi", - "type": "text" - }, - { - "name": "abstract", - "type": "long_text" - }, - { - "name": "field", - "type": "select", - "options": ["Computer Science", "Biology", "Physics", "Chemistry", "Medicine", "Social Science"] - }, - { - "name": "read_status", - "type": "select", - "options": ["To Read", "Reading", "Read", "Referenced"], - "default": "To Read" - }, - { - "name": "tags", - "type": "multi_select", - "options": ["Database", "AI", "Formal Methods", "Distributed Systems", "Security"] - }, - { - "name": "prompt_provenance", - "type": "number", - "min": 0, - "max": 100, - "description": "Data source documentation score" - }, - { - "name": "prompt_reproducibility", - "type": "number", - "min": 0, - "max": 100, - "description": "Can results be recreated?" - }, - { - "name": "prompt_openness", - "type": "number", - "min": 0, - "max": 100, - "description": "Data and code availability" - }, - { - "name": "prompt_methodology", - "type": "number", - "min": 0, - "max": 100, - "description": "Process documentation quality" - }, - { - "name": "prompt_peer_review", - "type": "number", - "min": 0, - "max": 100, - "description": "Peer review and validation" - }, - { - "name": "prompt_timeliness", - "type": "number", - "min": 0, - "max": 100, - "description": "Data currency and relevance" - }, - { - "name": "prompt_total", - "type": "formula", - "formula": "AVERAGE({prompt_provenance}, {prompt_reproducibility}, {prompt_openness}, {prompt_methodology}, {prompt_peer_review}, {prompt_timeliness})" - }, - { - "name": "notes", - "type": "long_text" - }, - { - "name": "pdf", - "type": "attachment" - } - ] - }, - "records": [ - { - "title": "Dynamo: Amazon's Highly Available Key-value Store", - "authors": "DeCandia et al.", - "year": 2007, - "journal": "SOSP", - "doi": "10.1145/1294261.1294281", - "abstract": "Reliability at massive scale is one of the biggest challenges we face at Amazon.com...", - "field": "Computer Science", - "read_status": "Read", - "tags": ["Database", "Distributed Systems"], - "prompt_provenance": 100, - "prompt_reproducibility": 70, - "prompt_openness": 60, - "prompt_methodology": 95, - "prompt_peer_review": 100, - "prompt_timeliness": 50, - "notes": "Foundational paper on eventual consistency and distributed databases" - }, - { - "title": "The Lean Theorem Prover", - "authors": "de Moura & Ullrich", - "year": 2021, - "journal": "CPP", - "doi": "10.1145/3437992.3439919", - "abstract": "Lean is a proof assistant and functional programming language...", - "field": "Computer Science", - "read_status": "Referenced", - "tags": ["Formal Methods", "AI"], - "prompt_provenance": 100, - "prompt_reproducibility": 95, - "prompt_openness": 100, - "prompt_methodology": 100, - "prompt_peer_review": 100, - "prompt_timeliness": 95, - "notes": "Used for our dependent-types query language (GQL-DT)" - }, - { - "title": "Self-Normalizing Databases", - "authors": "Jewell, J.D.A.", - "year": 2026, - "journal": "arXiv", - "field": "Computer Science", - "read_status": "Reading", - "tags": ["Database", "Formal Methods"], - "prompt_provenance": 100, - "prompt_reproducibility": 90, - "prompt_openness": 100, - "prompt_methodology": 95, - "prompt_peer_review": 70, - "prompt_timeliness": 100, - "notes": "Our paper on Lithoglyph's automatic normalization" - } - ] - } - ], - "views": [ - { - "name": "High Quality Papers", - "type": "grid", - "filters": [{"field": "prompt_total", "operator": ">=", "value": 80}], - "sort": [{"field": "prompt_total", "direction": "desc"}] - }, - { - "name": "Reading List", - "type": "grid", - "filters": [{"field": "read_status", "operator": "in", "value": ["To Read", "Reading"]}], - "sort": [{"field": "year", "direction": "desc"}] - }, - { - "name": "By Field", - "type": "grid", - "group_by": "field" - } - ] -} diff --git a/lithoglyph/glyphbase/examples/task-tracker.json b/lithoglyph/glyphbase/examples/task-tracker.json deleted file mode 100644 index aa9e271e..00000000 --- a/lithoglyph/glyphbase/examples/task-tracker.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "name": "Task Tracker", - "description": "Simple task management with status, priority, and assignee tracking", - "collections": [ - { - "name": "tasks", - "schema": { - "fields": [ - { - "name": "title", - "type": "text", - "required": true - }, - { - "name": "description", - "type": "long_text" - }, - { - "name": "status", - "type": "select", - "options": ["Todo", "In Progress", "Review", "Done"], - "default": "Todo" - }, - { - "name": "priority", - "type": "select", - "options": ["Low", "Medium", "High", "Urgent"], - "default": "Medium" - }, - { - "name": "assignee", - "type": "text" - }, - { - "name": "due_date", - "type": "date" - }, - { - "name": "tags", - "type": "multi_select", - "options": ["Bug", "Feature", "Documentation", "Testing", "Design"] - }, - { - "name": "prompt_score", - "type": "number", - "min": 0, - "max": 100 - } - ] - }, - "records": [ - { - "title": "Set up Glyphbase", - "description": "Install and configure Glyphbase for the team", - "status": "Done", - "priority": "High", - "assignee": "Alice", - "due_date": "2026-02-01", - "tags": ["Documentation"], - "prompt_score": 95 - }, - { - "title": "Create user guide", - "description": "Write comprehensive user documentation", - "status": "In Progress", - "priority": "High", - "assignee": "Bob", - "due_date": "2026-02-15", - "tags": ["Documentation"], - "prompt_score": 85 - }, - { - "title": "Fix login bug", - "description": "Users can't log in after password reset", - "status": "Todo", - "priority": "Urgent", - "assignee": "Charlie", - "due_date": "2026-02-07", - "tags": ["Bug"], - "prompt_score": 90 - }, - { - "title": "Add dark mode", - "description": "Implement dark mode theme option", - "status": "Todo", - "priority": "Low", - "assignee": "Diana", - "due_date": "2026-03-01", - "tags": ["Feature", "Design"], - "prompt_score": 75 - } - ] - } - ], - "views": [ - { - "name": "Kanban Board", - "type": "kanban", - "group_by": "status" - }, - { - "name": "By Priority", - "type": "grid", - "sort": [{"field": "priority", "direction": "desc"}], - "filters": [{"field": "status", "operator": "!=", "value": "Done"}] - }, - { - "name": "Calendar", - "type": "calendar", - "date_field": "due_date" - } - ] -} diff --git a/lithoglyph/glyphbase/examples/web-project-deno.json b/lithoglyph/glyphbase/examples/web-project-deno.json deleted file mode 100644 index 5ddd3bd7..00000000 --- a/lithoglyph/glyphbase/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "^12.0.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/lithoglyph/glyphbase/ffi/zig/build.zig b/lithoglyph/glyphbase/ffi/zig/build.zig deleted file mode 100644 index 4a2e049a..00000000 --- a/lithoglyph/glyphbase/ffi/zig/build.zig +++ /dev/null @@ -1,94 +0,0 @@ -// {{PROJECT}} FFI Build Configuration -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Shared library (.so, .dylib, .dll) - const lib = b.addSharedLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Set version - lib.version = .{ .major = 0, .minor = 1, .patch = 0 }; - - // Static library (.a) - const lib_static = b.addStaticLibrary(.{ - .name = "{{project}}", - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - // Install artifacts - b.installArtifact(lib); - b.installArtifact(lib_static); - - // Generate header file for C compatibility - const header = b.addInstallHeader( - b.path("include/{{project}}.h"), - "{{project}}.h", - ); - b.getInstallStep().dependOn(&header.step); - - // Unit tests - const lib_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const run_lib_tests = b.addRunArtifact(lib_tests); - - const test_step = b.step("test", "Run library tests"); - test_step.dependOn(&run_lib_tests.step); - - // Integration tests - const integration_tests = b.addTest(.{ - .root_source_file = b.path("test/integration_test.zig"), - .target = target, - .optimize = optimize, - }); - - integration_tests.linkLibrary(lib); - - const run_integration_tests = b.addRunArtifact(integration_tests); - - const integration_test_step = b.step("test-integration", "Run integration tests"); - integration_test_step.dependOn(&run_integration_tests.step); - - // Documentation - const docs = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = .Debug, - }); - - const docs_step = b.step("docs", "Generate documentation"); - docs_step.dependOn(&b.addInstallDirectory(.{ - .source_dir = docs.getEmittedDocs(), - .install_dir = .prefix, - .install_subdir = "docs", - }).step); - - // Benchmark (if needed) - const bench = b.addExecutable(.{ - .name = "{{project}}-bench", - .root_source_file = b.path("bench/bench.zig"), - .target = target, - .optimize = .ReleaseFast, - }); - - bench.linkLibrary(lib); - - const run_bench = b.addRunArtifact(bench); - - const bench_step = b.step("bench", "Run benchmarks"); - bench_step.dependOn(&run_bench.step); -} diff --git a/lithoglyph/glyphbase/ffi/zig/src/main.zig b/lithoglyph/glyphbase/ffi/zig/src/main.zig deleted file mode 100644 index 6b233bc7..00000000 --- a/lithoglyph/glyphbase/ffi/zig/src/main.zig +++ /dev/null @@ -1,274 +0,0 @@ -// {{PROJECT}} FFI Implementation -// -// This module implements the C-compatible FFI declared in src/abi/Foreign.idr -// All types and layouts must match the Idris2 ABI definitions. -// -// SPDX-License-Identifier: MPL-2.0 - -const std = @import("std"); - -// Version information (keep in sync with project) -const VERSION = "0.1.0"; -const BUILD_INFO = "{{PROJECT}} built with Zig " ++ @import("builtin").zig_version_string; - -/// Thread-local error storage -threadlocal var last_error: ?[]const u8 = null; - -/// Set the last error message -fn setError(msg: []const u8) void { - last_error = msg; -} - -/// Clear the last error -fn clearError() void { - last_error = null; -} - -//============================================================================== -// Core Types (must match src/abi/Types.idr) -//============================================================================== - -/// Result codes (must match Idris2 Result type) -pub const Result = enum(c_int) { - ok = 0, - @"error" = 1, - invalid_param = 2, - out_of_memory = 3, - null_pointer = 4, -}; - -/// Library handle (opaque to prevent direct access) -pub const Handle = opaque { - // Internal state hidden from C - allocator: std.mem.Allocator, - initialized: bool, - // Add your fields here -}; - -//============================================================================== -// Library Lifecycle -//============================================================================== - -/// Initialize the library -/// Returns a handle, or null on failure -export fn {{project}}_init() ?*Handle { - const allocator = std.heap.c_allocator; - - const handle = allocator.create(Handle) catch { - setError("Failed to allocate handle"); - return null; - }; - - // Initialize handle - handle.* = .{ - .allocator = allocator, - .initialized = true, - }; - - clearError(); - return handle; -} - -/// Free the library handle -export fn {{project}}_free(handle: ?*Handle) void { - const h = handle orelse return; - const allocator = h.allocator; - - // Clean up resources - h.initialized = false; - - allocator.destroy(h); - clearError(); -} - -//============================================================================== -// Core Operations -//============================================================================== - -/// Process data (example operation) -export fn {{project}}_process(handle: ?*Handle, input: u32) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Example processing logic - _ = input; - - clearError(); - return .ok; -} - -//============================================================================== -// String Operations -//============================================================================== - -/// Get a string result (example) -/// Caller must free the returned string -export fn {{project}}_get_string(handle: ?*Handle) ?[*:0]const u8 { - const h = handle orelse { - setError("Null handle"); - return null; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return null; - } - - // Example: allocate and return a string - const result = h.allocator.dupeZ(u8, "Example result") catch { - setError("Failed to allocate string"); - return null; - }; - - clearError(); - return result.ptr; -} - -/// Free a string allocated by the library -export fn {{project}}_free_string(str: ?[*:0]const u8) void { - const s = str orelse return; - const allocator = std.heap.c_allocator; - - const slice = std.mem.span(s); - allocator.free(slice); -} - -//============================================================================== -// Array/Buffer Operations -//============================================================================== - -/// Process an array of data -export fn {{project}}_process_array( - handle: ?*Handle, - buffer: ?[*]const u8, - len: u32, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const buf = buffer orelse { - setError("Null buffer"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Access the buffer - const data = buf[0..len]; - _ = data; - - // Process data here - - clearError(); - return .ok; -} - -//============================================================================== -// Error Handling -//============================================================================== - -/// Get the last error message -/// Returns null if no error -export fn {{project}}_last_error() ?[*:0]const u8 { - const err = last_error orelse return null; - - // Return C string (static storage, no need to free) - const allocator = std.heap.c_allocator; - const c_str = allocator.dupeZ(u8, err) catch return null; - return c_str.ptr; -} - -//============================================================================== -// Version Information -//============================================================================== - -/// Get the library version -export fn {{project}}_version() [*:0]const u8 { - return VERSION.ptr; -} - -/// Get build information -export fn {{project}}_build_info() [*:0]const u8 { - return BUILD_INFO.ptr; -} - -//============================================================================== -// Callback Support -//============================================================================== - -/// Callback function type (C ABI) -pub const Callback = *const fn (u64, u32) callconv(.C) u32; - -/// Register a callback -export fn {{project}}_register_callback( - handle: ?*Handle, - callback: ?Callback, -) Result { - const h = handle orelse { - setError("Null handle"); - return .null_pointer; - }; - - const cb = callback orelse { - setError("Null callback"); - return .null_pointer; - }; - - if (!h.initialized) { - setError("Handle not initialized"); - return .@"error"; - } - - // Store callback for later use - _ = cb; - - clearError(); - return .ok; -} - -//============================================================================== -// Utility Functions -//============================================================================== - -/// Check if handle is initialized -export fn {{project}}_is_initialized(handle: ?*Handle) u32 { - const h = handle orelse return 0; - return if (h.initialized) 1 else 0; -} - -//============================================================================== -// Tests -//============================================================================== - -test "lifecycle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try std.testing.expect({{project}}_is_initialized(handle) == 1); -} - -test "error handling" { - const result = {{project}}_process(null, 0); - try std.testing.expectEqual(Result.null_pointer, result); - - const err = {{project}}_last_error(); - try std.testing.expect(err != null); -} - -test "version" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - try std.testing.expectEqualStrings(VERSION, ver_str); -} diff --git a/lithoglyph/glyphbase/ffi/zig/test/integration_test.zig b/lithoglyph/glyphbase/ffi/zig/test/integration_test.zig deleted file mode 100644 index 03419949..00000000 --- a/lithoglyph/glyphbase/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,182 +0,0 @@ -// {{PROJECT}} Integration Tests -// SPDX-License-Identifier: MPL-2.0 -// -// These tests verify that the Zig FFI correctly implements the Idris2 ABI - -const std = @import("std"); -const testing = std.testing; - -// Import FFI functions -extern fn {{project}}_init() ?*opaque {}; -extern fn {{project}}_free(?*opaque {}) void; -extern fn {{project}}_process(?*opaque {}, u32) c_int; -extern fn {{project}}_get_string(?*opaque {}) ?[*:0]const u8; -extern fn {{project}}_free_string(?[*:0]const u8) void; -extern fn {{project}}_last_error() ?[*:0]const u8; -extern fn {{project}}_version() [*:0]const u8; -extern fn {{project}}_is_initialized(?*opaque {}) u32; - -//============================================================================== -// Lifecycle Tests -//============================================================================== - -test "create and destroy handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - try testing.expect(handle != null); -} - -test "handle is initialized" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const initialized = {{project}}_is_initialized(handle); - try testing.expectEqual(@as(u32, 1), initialized); -} - -test "null handle is not initialized" { - const initialized = {{project}}_is_initialized(null); - try testing.expectEqual(@as(u32, 0), initialized); -} - -//============================================================================== -// Operation Tests -//============================================================================== - -test "process with valid handle" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const result = {{project}}_process(handle, 42); - try testing.expectEqual(@as(c_int, 0), result); // 0 = ok -} - -test "process with null handle returns error" { - const result = {{project}}_process(null, 42); - try testing.expectEqual(@as(c_int, 4), result); // 4 = null_pointer -} - -//============================================================================== -// String Tests -//============================================================================== - -test "get string result" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const str = {{project}}_get_string(handle); - defer if (str) |s| {{project}}_free_string(s); - - try testing.expect(str != null); -} - -test "get string with null handle" { - const str = {{project}}_get_string(null); - try testing.expect(str == null); -} - -//============================================================================== -// Error Handling Tests -//============================================================================== - -test "last error after null handle operation" { - _ = {{project}}_process(null, 0); - - const err = {{project}}_last_error(); - try testing.expect(err != null); - - if (err) |e| { - const err_str = std.mem.span(e); - try testing.expect(err_str.len > 0); - } -} - -test "no error after successful operation" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - _ = {{project}}_process(handle, 0); - - // Error should be cleared after successful operation - // (This depends on implementation) -} - -//============================================================================== -// Version Tests -//============================================================================== - -test "version string is not empty" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - try testing.expect(ver_str.len > 0); -} - -test "version string is semantic version format" { - const ver = {{project}}_version(); - const ver_str = std.mem.span(ver); - - // Should be in format X.Y.Z - try testing.expect(std.mem.count(u8, ver_str, ".") >= 1); -} - -//============================================================================== -// Memory Safety Tests -//============================================================================== - -test "multiple handles are independent" { - const h1 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h1); - - const h2 = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(h2); - - try testing.expect(h1 != h2); - - // Operations on h1 should not affect h2 - _ = {{project}}_process(h1, 1); - _ = {{project}}_process(h2, 2); -} - -test "double free is safe" { - const handle = {{project}}_init() orelse return error.InitFailed; - - {{project}}_free(handle); - {{project}}_free(handle); // Should not crash -} - -test "free null is safe" { - {{project}}_free(null); // Should not crash -} - -//============================================================================== -// Thread Safety Tests (if applicable) -//============================================================================== - -test "concurrent operations" { - const handle = {{project}}_init() orelse return error.InitFailed; - defer {{project}}_free(handle); - - const ThreadContext = struct { - h: *opaque {}, - id: u32, - }; - - const thread_fn = struct { - fn run(ctx: ThreadContext) void { - _ = {{project}}_process(ctx.h, ctx.id); - } - }.run; - - var threads: [4]std.Thread = undefined; - for (&threads, 0..) |*thread, i| { - thread.* = try std.Thread.spawn(.{}, thread_fn, .{ - ThreadContext{ .h = handle, .id = @intCast(i) }, - }); - } - - for (threads) |thread| { - thread.join(); - } -} diff --git a/lithoglyph/glyphbase/scripts/rebrand-to-lithoglyph.sh b/lithoglyph/glyphbase/scripts/rebrand-to-lithoglyph.sh deleted file mode 100755 index adc47f66..00000000 --- a/lithoglyph/glyphbase/scripts/rebrand-to-lithoglyph.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# Comprehensive Glyphbase Rebrand Script -# Renames Lith/Lith → Lithoglyph throughout codebase - -set -euo pipefail - -echo "=== Glyphbase Comprehensive Rebrand Script ===" -echo "Lith/Lith → Lithoglyph" -echo "formbase → glyphbase (where needed)" -echo "" - -# Count changes -TOTAL_FILES=0 -TOTAL_CHANGES=0 - -# Function to rebrand a single file -rebrand_file() { - local file="$1" - if [ ! -f "$file" ]; then - return - fi - - # Skip backup files - if [[ "$file" == *.bak ]]; then - return - fi - - # Create backup - cp "$file" "$file.bak" - - # Count lines before - local before=$(wc -l < "$file" 2>/dev/null || echo "0") - - # Perform replacements - sed -i \ - -e 's/Lith/Lithoglyph/g' \ - -e 's/lith/lithoglyph/g' \ - -e 's/Lith/Lithoglyph/g' \ - -e 's/lith/lithoglyph/g' \ - -e 's/FormBase/Glyphbase/g' \ - -e 's/formbase/glyphbase/g' \ - -e 's/FORMBD/LITHOGLYPH/g' \ - -e 's/FORMDB/LITHOGLYPH/g' \ - "$file" - - # Check if file changed - if ! diff -q "$file" "$file.bak" > /dev/null 2>&1; then - local changed=$(diff "$file.bak" "$file" | grep -c '^[<>]' || echo "0") - echo "✓ $file ($changed lines changed)" - TOTAL_FILES=$((TOTAL_FILES + 1)) - TOTAL_CHANGES=$((TOTAL_CHANGES + changed)) - else - # No changes, remove backup - rm "$file.bak" - fi -} - -# Rebrand source files -echo "Rebranding Gleam source files..." -for file in server/src/**/*.gleam; do - rebrand_file "$file" -done - -# Rebrand Erlang NIF files -echo "" -echo "Rebranding Erlang NIF files..." -for file in server/src/*.erl server/native/src/*.erl server/*.erl; do - rebrand_file "$file" -done - -# Rebrand Zig FFI files -echo "" -echo "Rebranding Zig FFI files..." -for file in server/ffi/zig/src/*.zig server/ffi/zig/*.zig server/native/src/*.zig; do - rebrand_file "$file" -done - -# Rebrand Idris ABI files -echo "" -echo "Rebranding Idris ABI files..." -for file in server/src/abi/*.idr; do - rebrand_file "$file" -done - -# Rebrand Rust files -echo "" -echo "Rebranding Rust files..." -for file in server/native_rust/src/*.rs server/native_rust/*.toml; do - rebrand_file "$file" -done - -# Rebrand ReScript UI files -echo "" -echo "Rebranding ReScript UI files..." -for file in ui/src/**/*.res; do - rebrand_file "$file" -done - -# Rebrand documentation -echo "" -echo "Rebranding documentation..." -for file in *.md *.adoc docs/*.md docs/*.adoc server/*.md server/ffi/zig/*.md; do - rebrand_file "$file" -done - -# Rebrand test files -echo "" -echo "Rebranding test files..." -for file in server/test/*.gleam; do - rebrand_file "$file" -done - -# Rebrand build files -echo "" -echo "Rebranding build files..." -for file in Justfile docker-compose.yml server/ffi/zig/build.zig server/native/build.zig server/native/Makefile; do - rebrand_file "$file" -done - -# Rebrand UI config files -echo "" -echo "Rebranding UI config files..." -for file in ui/package.json ui/rescript.json; do - rebrand_file "$file" -done - -# Rebrand CI workflows -echo "" -echo "Rebranding CI workflows..." -for file in .github/workflows/*.yml; do - rebrand_file "$file" -done - -echo "" -echo "=== File Renames Required ===" -echo "The following files should be renamed manually:" -echo "" - -# Check for files that need renaming -if [ -f "server/src/formbase_server.gleam" ]; then - echo " server/src/formbase_server.gleam → server/src/glyphbase_server.gleam" -fi - -if [ -f "server/src/lith.gleam" ]; then - echo " server/src/lith.gleam → server/src/lithoglyph.gleam" -fi - -if [ -f "server/src/lith_nif.erl" ]; then - echo " server/src/lith_nif.erl → server/src/lithoglyph_nif.erl" -fi - -if [ -f "server/src/lith_nif.erl" ]; then - echo " server/src/lith_nif.erl → server/src/lithoglyph_nif.erl.old" -fi - -if [ -f "server/native/src/lith_nif.erl" ]; then - echo " server/native/src/lith_nif.erl → server/native/src/lithoglyph_nif.erl" -fi - -if [ -f "server/native/src/lith_nif.zig" ]; then - echo " server/native/src/lith_nif.zig → server/native/src/lithoglyph_nif.zig" -fi - -if [ -f "server/test_lith_nif.erl" ]; then - echo " server/test_lith_nif.erl → server/test_lithoglyph_nif.erl" -fi - -if [ -f "FORMBD-INTEGRATION.md" ]; then - echo " FORMBD-INTEGRATION.md → LITHOGLYPH-INTEGRATION.md" -fi - -if [ -d "server/src/lith" ]; then - echo " server/src/lith/ → server/src/lithoglyph/" -fi - -echo "" -echo "=== Summary ===" -echo "Files modified: $TOTAL_FILES" -echo "Lines changed: $TOTAL_CHANGES" -echo "" -echo "Next steps:" -echo "1. Review changes with: git diff" -echo "2. Rename files listed above" -echo "3. Update import statements to use new names" -echo "4. Test build: cd server && gleam build" -echo "5. Commit changes: git commit -am 'feat: rebrand Lith/Lith to Lithoglyph'" diff --git a/lithoglyph/glyphbase/selur-compose.yml b/lithoglyph/glyphbase/selur-compose.yml deleted file mode 100644 index a17be13f..00000000 --- a/lithoglyph/glyphbase/selur-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Glyphbase Docker Compose Configuration - -services: - glyphbase: - image: ghcr.io/hyperpolymath/glyphbase:latest - container_name: glyphbase - ports: - - "4000:4000" - volumes: - - ./data:/data - environment: - - PORT=4000 - - DATABASE_PATH=/data - - SECRET_KEY=${SECRET_KEY:-change-me-in-production} - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:4000/health"] - interval: 30s - timeout: 3s - retries: 3 - start_period: 5s - - # Optional: Lithoglyph standalone database - lithoglyph: - image: ghcr.io/hyperpolymath/lithoglyph:latest - container_name: lithoglyph-db - ports: - - "5432:5432" - volumes: - - lithoglyph-data:/var/lib/lithoglyph - environment: - - LITHOGLYPH_DATA_DIR=/var/lib/lithoglyph - restart: unless-stopped - profiles: - - full - -volumes: - lithoglyph-data: - driver: local diff --git a/lithoglyph/glyphbase/server/.gitkeep b/lithoglyph/glyphbase/server/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/lithoglyph/glyphbase/server/ABI-FFI-README.md b/lithoglyph/glyphbase/server/ABI-FFI-README.md deleted file mode 100644 index 77984cd8..00000000 --- a/lithoglyph/glyphbase/server/ABI-FFI-README.md +++ /dev/null @@ -1,180 +0,0 @@ -# Lith ABI/FFI Architecture - -This document describes the **Idris2 ABI + Zig FFI** architecture used for Lith/Lithoglyph database integration. - -## Architecture Overview - -Following the **hyperpolymath ABI/FFI Universal Standard**, this codebase uses: - -| Layer | Language | Purpose | Location | -|-------|----------|---------|----------| -| **ABI** | **Idris2** | Interface definitions with formal proofs | `src/abi/*.idr` | -| **FFI** | **Zig** | C-compatible implementation | `ffi/zig/src/*.zig` | -| **Headers** | C (generated) | Bridge between ABI and FFI | `generated/abi/*.h` | - -## Why This Architecture? - -### Idris2 for ABI - -- **Dependent types** prove interface correctness at compile-time -- **Formal verification** of memory layout (alignment, padding, size) -- **Platform-specific ABIs** with compile-time selection -- **Provable backward compatibility** between versions -- **Type-level guarantees** impossible in C/Zig/Rust -- **Self-documenting** with mathematical proofs - -### Zig for FFI - -- **Native C ABI compatibility** without overhead -- **Memory-safe by default** -- **Cross-compilation built-in** (any platform, any architecture) -- **No runtime dependencies** -- **Zero-cost abstractions** -- **Better error handling** than C, simpler than Rust FFI - -## Directory Structure - -``` -server/ -├── src/ -│ ├── abi/ # Idris2 ABI definitions -│ │ ├── Types.idr # Type definitions with proofs -│ │ ├── Layout.idr # Memory layout verification -│ │ └── Foreign.idr # FFI declarations -│ └── lith/ # Gleam client wrapper -│ ├── client.gleam # High-level API -│ └── nif_ffi.gleam # Low-level NIF bindings -│ -├── ffi/ -│ └── zig/ # Zig FFI implementation -│ ├── build.zig # Build script -│ ├── src/ -│ │ └── main.zig # C-compatible implementation -│ └── test/ -│ └── integration_test.zig -│ -└── priv/ # Compiled NIF libraries - └── lith_nif.so # Built from Zig -``` - -## API Surface - -### Core Types - -```idris --- Non-null database handle (proven at type level) -data DbHandle : Type where - MkDbHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> DbHandle - --- Non-null transaction handle (proven at type level) -data TxnHandle : Type where - MkTxnHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> TxnHandle - --- Transaction mode -data TxnMode = ReadOnly | ReadWrite - --- Result type (matches Erlang {ok, Value} | {error, Reason}) -data FFIResult a = Ok a | Error String -``` - -### Core Functions - -```idris --- Get NIF version -nifVersion : IO Version - --- Database operations -dbOpen : DbPath -> IO (FFIResult DbHandle) -dbClose : DbHandle -> IO (FFIResult ()) - --- Transaction operations -txnBegin : DbHandle -> TxnMode -> IO (FFIResult TxnHandle) -txnCommit : TxnHandle -> IO (FFIResult ()) -txnAbort : TxnHandle -> IO (FFIResult ()) - --- Database operations -applyOperation : TxnHandle -> OperationData -> IO (FFIResult (BlockId, Maybe (List Bits8))) -getSchema : DbHandle -> IO (FFIResult SchemaData) -getJournal : DbHandle -> Timestamp -> IO (FFIResult JournalData) -``` - -## Memory Layout Guarantees - -The `Layout.idr` module provides compile-time proofs: - -1. **Pointer sizes** are 8 bytes on all supported platforms -2. **Pointer alignment** matches size (8-byte aligned) -3. **Handle types** are always pointer-sized (stable ABI) -4. **No padding** in Version struct (3 bytes total) -5. **Cross-platform compatibility** (Linux, macOS, Windows on x86_64/ARM64) - -## Building - -### Build Zig NIF - -```bash -cd ffi/zig -zig build -# Output: ../../priv/liblith_nif.so -``` - -### Run Tests - -```bash -cd ffi/zig -zig build test # Unit tests -zig build test-integration # Integration tests -``` - -### Verify Idris2 ABI - -```bash -cd src/abi -idris2 --check Types.idr -idris2 --check Layout.idr -idris2 --check Foreign.idr -``` - -## Integration with Gleam - -The Gleam client (`src/lith/client.gleam`) wraps the NIF functions: - -```gleam -pub fn connect(path: String) -> LithResult(Connection) { - let path_binary = bit_array.from_string(path) - let handle = nif_ffi.nif_db_open(path_binary) - Ok(Connection(handle: handle)) -} - -pub fn begin_transaction(conn: Connection, mode: TransactionMode) -> LithResult(Transaction) { - let mode_binary = transaction_mode_to_binary(mode) - let result = nif_ffi.nif_txn_begin(conn.handle, mode_binary) - // ... handle Erlang {ok, Handle} | {error, Reason} tuples -} -``` - -## TODO: Integration with Lithoglyph - -Currently, the Zig FFI contains **placeholder implementations**. To integrate with the real Lithoglyph database: - -1. **Add Lithoglyph dependency** to `ffi/zig/build.zig` -2. **Replace placeholder structs** in `main.zig` with real Lithoglyph handles -3. **Implement CBOR parsing** for operations -4. **Call Lithoglyph C API** from Zig functions -5. **Add provenance tracking** integration - -## Proofs Required - -Every ABI must prove (see `Types.idr` and `Layout.idr`): - -1. ✅ **Type Safety**: Opaque handles prevent null pointers -2. ✅ **Layout Correctness**: Struct size and alignment match platform -3. ✅ **Platform Compatibility**: Same ABI works on all platforms -4. ⏳ **Version Compatibility**: New versions don't break old ABIs (WIP) - -## See Also - -- [Hyperpolymath ABI/FFI Standard](~/.claude/CLAUDE.md#abi-ffi-universal-standard) -- [RSR Template ABI/FFI](~/Documents/hyperpolymath-repos/rsr-template-repo/ABI-FFI-README.md) -- [Proven Library](~/Documents/hyperpolymath-repos/proven) - Idris2 proofs library -- [Ephapax](~/Documents/hyperpolymath-repos/ephapax) - Reference Idris2 + Zig FFI implementation diff --git a/lithoglyph/glyphbase/server/ffi/zig/BUILD-ISSUES.md b/lithoglyph/glyphbase/server/ffi/zig/BUILD-ISSUES.md deleted file mode 100644 index 5e477920..00000000 --- a/lithoglyph/glyphbase/server/ffi/zig/BUILD-ISSUES.md +++ /dev/null @@ -1,81 +0,0 @@ -# Build Issues (Zig 0.15.2) - -## Current Blockers - -### 1. Erlang NIF Headers Not Found -``` -error: 'erl_nif.h' file not found -``` - -**Solution**: Install Erlang development package: -```bash -# Fedora -sudo dnf install erlang-devel - -# Ubuntu -sudo apt-get install erlang-dev - -# Or set ERTS_INCLUDE_DIR -export ERTS_INCLUDE_DIR=/usr/lib/erlang/erts-*/include -``` - -### 2. Lithoglyph Functions Not Public -``` -error: 'lith_db_open' is not marked 'pub' -``` - -The `export` functions in Lithoglyph bridge.zig are C-exported but not Zig-public. - -**Solution**: Either: -- Use C FFI to call them (via `@extern`) -- Add `pub` before `export` in Lithoglyph bridge.zig -- Link against compiled Lithoglyph library instead of importing source - -### 3. Calling Convention API Change -``` -error: union 'builtin.CallingConvention' has no member named 'C' -``` - -Zig 0.15.2 changed calling convention naming. - -**Solution**: Check Lithoglyph compatibility with Zig 0.15.2. - -## Workaround: Use C FFI - -Instead of importing Lithoglyph as a module, we can link against it as a C library: - -```zig -// Declare C functions -extern fn lith_db_open( - path_ptr: [*]const u8, - path_len: usize, - opts_ptr: ?[*]const u8, - opts_len: usize, - out_db: *?*LithDb, - out_err: *LithBlob, -) LithStatus; - -// Use them directly -const status = lith_db_open(path.ptr, path.len, null, 0, &out_db, &out_err); -``` - -Then link: -```zig -lib.linkSystemLibrary("lith"); -lib.addLibraryPath(.{ .cwd_relative = lithoglyph_path ++ "/zig-out/lib" }); -``` - -## Status - -- ✅ Idris2 ABI complete (formal proofs) -- ✅ Zig FFI structure complete (all 9 functions) -- ✅ Lithoglyph integration code written -- ❌ Build blocked by environment/API issues -- ⏸️ Paused pending resolution - -## Next Steps - -1. Install Erlang dev headers -2. Either fix Lithoglyph pub exports or use C FFI approach -3. Test Lithoglyph compatibility with Zig 0.15.2 -4. Complete build and integration tests diff --git a/lithoglyph/glyphbase/server/ffi/zig/README.md b/lithoglyph/glyphbase/server/ffi/zig/README.md deleted file mode 100644 index ef2911b2..00000000 --- a/lithoglyph/glyphbase/server/ffi/zig/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Glyphbase Zig FFI - Lithoglyph Integration - -This directory contains the Zig FFI implementation that integrates Glyphbase with the Lithoglyph database. - -## Architecture - -Following the **hyperpolymath ABI/FFI Universal Standard**: - -- **ABI Layer**: `../../../src/abi/*.idr` (Idris2 with formal proofs) -- **FFI Layer**: This directory (Zig with C-compatible exports) -- **Database Engine**: Lithoglyph core-zig (imported as dependency) - -## Building - -### Prerequisites - -1. **Zig** (0.13.0 or later) -2. **Lithoglyph repository** cloned at `~/Documents/hyperpolymath-repos/lithoglyph` -3. **Erlang/OTP** (for NIF headers) - -### Environment Variables - -```bash -# Optional: Override Lithoglyph path -export LITHOGLYPH_PATH=~/Documents/hyperpolymath-repos/lithoglyph/lith/database/core-zig - -# Optional: Override ERTS include directory -export ERTS_INCLUDE_DIR=/usr/lib/erlang/usr/include -``` - -### Build Commands - -```bash -# Build the NIF shared library -zig build - -# Output: ../../priv/liblith_nif.so (Linux) -# ../../priv/liblith_nif.dylib (macOS) -# ../../priv/lith_nif.dll (Windows) - -# Run unit tests -zig build test - -# Run integration tests -zig build test-integration -``` - -## Integration with Gleam - -The compiled NIF library is loaded by the Gleam server via `src/lith/nif_ffi.gleam`: - -```gleam -@external(erlang, "lith_nif", "db_open") -pub fn nif_db_open(path: BitArray) -> DbHandle -``` - -## API Functions - -All functions are exported with C calling convention: - -| Function | Description | -|----------|-------------| -| `lith_nif_version` | Get NIF version (0.1.0) | -| `lith_nif_db_open` | Open database connection | -| `lith_nif_db_close` | Close database connection | -| `lith_nif_txn_begin` | Begin transaction (read-only or read-write) | -| `lith_nif_txn_commit` | Commit transaction | -| `lith_nif_txn_abort` | Abort transaction | -| `lith_nif_apply` | Apply CBOR-encoded operation | -| `lith_nif_schema` | Get database schema (CBOR-encoded) | -| `lith_nif_journal` | Get journal entries since timestamp | - -## Lithoglyph Core Integration - -This FFI wraps the Lithoglyph core-zig library (`lithoglyph/lith/database/core-zig`): - -```zig -const lithoglyph = @import("lithoglyph"); - -// Use Lithoglyph types -const LithDb = lithoglyph.LithDb; -const LithTxn = lithoglyph.LithTxn; -const LithStatus = lithoglyph.types.LithStatus; - -// Call Lithoglyph functions -const status = lithoglyph.lith_db_open( - path.ptr, - path.len, - null, // options - 0, // options_len - &out_db, - &out_err, -); -``` - -## Memory Management - -- **Allocator**: Uses `std.heap.GeneralPurposeAllocator` for all allocations -- **Handle Management**: Lithoglyph core-zig maintains handle registries -- **Error Blobs**: CBOR-encoded error messages allocated by Lithoglyph -- **Provenance Data**: Returned as CBOR blobs from operations - -## Error Handling - -All functions return status codes or NULL pointers on error: - -```zig -// Success: returns non-null pointer -const db = lith_nif_db_open("/path/to/database.ldb"); - -// Error: returns null -if (db == null) { - // Error logged via std.log.err -} -``` - -## CBOR Encoding - -Operations, results, and provenance use CBOR encoding: - -- **Operations**: `{"op": "insert", "collection": "docs", "data": {...}}` -- **Results**: `{"status": "ok", "doc_id": 123}` -- **Provenance**: `{"actor": "user", "timestamp": "...", "rationale": "..."}` - -## TODO - -- [ ] Implement schema retrieval (currently returns empty map) -- [ ] Implement journal retrieval (currently returns empty array) -- [ ] Add CBOR parsing for extracting block IDs from results -- [ ] Add comprehensive error handling with detailed error messages -- [ ] Add performance benchmarks -- [ ] Add fuzzing tests - -## See Also - -- [ABI Documentation](../../../src/abi/README.md) - Idris2 ABI with proofs -- [Lithoglyph Core](~/Documents/hyperpolymath-repos/lithoglyph/lith/database/core-zig) - Database engine -- [Server Integration](../../README.md) - Gleam server using this NIF diff --git a/lithoglyph/glyphbase/server/ffi/zig/build.zig b/lithoglyph/glyphbase/server/ffi/zig/build.zig deleted file mode 100644 index 22574cbd..00000000 --- a/lithoglyph/glyphbase/server/ffi/zig/build.zig +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Build script for Lithoglyph Zig FFI NIF library (Zig 0.15.2+) - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // Find Erlang NIF headers - const erts_include = std.process.getEnvVarOwned( - b.allocator, - "ERTS_INCLUDE_DIR" - ) catch blk: { - // Try to find via asdf - const home = std.process.getEnvVarOwned(b.allocator, "HOME") catch "$HOME"; - break :blk b.pathJoin(&.{home, ".asdf/installs/erlang/28.3.1/erts-16.2/include"}); - }; - - // Build shared library for Erlang NIF - const lib = b.addLibrary(.{ - .name = "lithoglyph_nif", - .linkage = .dynamic, - .root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - .link_libc = true, - }), - }); - - lib.addIncludePath(.{ .cwd_relative = erts_include }); - - // Note: Lithoglyph functions are declared as extern in main.zig - // They will be resolved at runtime when the NIF loads - // To link statically, uncomment: - // const lithoglyph_path = ...; - // lib.addLibraryPath(.{ .cwd_relative = lithoglyph_path ++ "/zig-out/lib" }); - // lib.linkSystemLibrary("lithoglyph"); - - // Install to priv directory for Erlang to find - const install_artifact = b.addInstallArtifact(lib, .{ - .dest_dir = .{ - .override = .{ - .custom = "../../priv", - }, - }, - }); - b.getInstallStep().dependOn(&install_artifact.step); - - // Unit tests - const main_tests = b.addTest(.{ - .root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }), - }); - - const run_main_tests = b.addRunArtifact(main_tests); - const test_step = b.step("test", "Run library tests"); - test_step.dependOn(&run_main_tests.step); -} diff --git a/lithoglyph/glyphbase/server/ffi/zig/src/main.zig b/lithoglyph/glyphbase/server/ffi/zig/src/main.zig deleted file mode 100644 index 6108b44c..00000000 --- a/lithoglyph/glyphbase/server/ffi/zig/src/main.zig +++ /dev/null @@ -1,488 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph NIF Implementation in Zig -// -// This implements the C-compatible ABI defined in ../../../src/abi/Foreign.idr -// All functions follow the memory layout guarantees from ../../../src/abi/Layout.idr - -const std = @import("std"); -const c = @cImport({ - @cInclude("erl_nif.h"); -}); - -// Opaque handle types (matching Idris2 ABI and Lithoglyph) -const LithDb = opaque {}; -const LithTxn = opaque {}; - -const DbHandle = LithDb; -const TxnHandle = LithTxn; - -// Lithoglyph types (C ABI compatible) -const LithStatus = enum(i32) { - ok = 0, - err_db_not_found = 1001, - err_db_already_open = 1002, - err_txn_not_active = 2001, - err_txn_already_committed = 2002, - err_invalid_argument = 9003, - err_internal = 9001, - err_out_of_memory = 9002, - err_not_implemented = 9004, -}; - -const LithTxnMode = enum(u8) { - read_only = 0, - read_write = 1, -}; - -const BlobEncoding = enum(u8) { - cbor = 0, - cbor_compressed = 1, - reserved = 255, -}; - -const LithBlob = extern struct { - data: ?[*]const u8, - len: usize, - encoding: BlobEncoding, - _padding: [7]u8, - - fn empty() LithBlob { - return .{ - .data = null, - .len = 0, - .encoding = .cbor, - ._padding = [_]u8{0} ** 7, - }; - } - - fn toSlice(self: LithBlob) ?[]const u8 { - if (self.data) |ptr| { - return ptr[0..self.len]; - } - return null; - } -}; - -const LithResult = extern struct { - result_blob: LithBlob, - provenance_blob: LithBlob, - status: LithStatus, - _padding: [4]u8, - err_blob: LithBlob, -}; - -// Extern declarations for Lithoglyph C API -extern fn lith_db_open( - path_ptr: [*]const u8, - path_len: usize, - opts_ptr: ?[*]const u8, - opts_len: usize, - out_db: *?*LithDb, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_db_close(db: ?*LithDb) LithStatus; - -extern fn lith_txn_begin( - db: ?*LithDb, - mode: LithTxnMode, - out_txn: *?*LithTxn, - out_err: *LithBlob, -) LithStatus; - -extern fn lith_txn_commit(txn: ?*LithTxn, out_err: *LithBlob) LithStatus; - -extern fn lith_txn_abort(txn: ?*LithTxn) LithStatus; - -extern fn lith_apply( - txn: ?*LithTxn, - op_ptr: [*]const u8, - op_len: usize, -) LithResult; - -// Version struct (matching Idris2 Version record) -const Version = extern struct { - major: u8, - minor: u8, - patch: u8, -}; - -// Block ID (u64) -const BlockId = u64; - -// Timestamp (u64 - Unix epoch microseconds) -const Timestamp = u64; - -// Global allocator for the NIF -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -const allocator = gpa.allocator(); - -// ============================================================================ -// Exported C-compatible functions (matching Idris2 Foreign.idr) -// ============================================================================ - -/// Get NIF version -export fn lithoglyph_nif_version(major: *u8, minor: *u8, patch: *u8) void { - // Lithoglyph NIF version 0.1.0 - major.* = 0; - minor.* = 1; - patch.* = 0; -} - -/// Open database connection -/// Returns: DbHandle pointer or NULL on error -export fn lithoglyph_nif_db_open(path: [*:0]const u8) ?*DbHandle { - const path_slice = std.mem.span(path); - - var out_db: ?*LithDb = null; - var out_err: LithBlob = undefined; - - const status = lith_db_open( - path_slice.ptr, - path_slice.len, - null, // opts_ptr - 0, // opts_len - &out_db, - &out_err, - ); - - if (status != .ok) { - // Log error if available - if (out_err.toSlice()) |err_slice| { - std.log.err("Failed to open database: {s}", .{err_slice}); - } - return null; - } - - return out_db; -} - -/// Close database connection -/// Returns: 0 on success, -1 on error -export fn lithoglyph_nif_db_close(handle: *DbHandle) c_int { - const status = lith_db_close(handle); - return if (status == .ok) 0 else -1; -} - -/// Begin transaction -/// mode: 0 = ReadOnly, 1 = ReadWrite -/// Returns: TxnHandle pointer or NULL on error -export fn lithoglyph_nif_txn_begin(handle: *DbHandle, mode_int: u32) ?*TxnHandle { - const mode: LithTxnMode = if (mode_int == 0) .read_only else .read_write; - - var out_txn: ?*LithTxn = null; - var out_err: LithBlob = undefined; - - const status = lith_txn_begin( - handle, - mode, - &out_txn, - &out_err, - ); - - if (status != .ok) { - if (out_err.toSlice()) |err_slice| { - std.log.err("Failed to begin transaction: {s}", .{err_slice}); - } - return null; - } - - return out_txn; -} - -/// Commit transaction -/// Returns: 0 on success, -1 on error -export fn lithoglyph_nif_txn_commit(handle: *TxnHandle) c_int { - var out_err: LithBlob = undefined; - const status = lith_txn_commit(handle, &out_err); - - if (status != .ok) { - if (out_err.toSlice()) |err_slice| { - std.log.err("Failed to commit transaction: {s}", .{err_slice}); - } - return -1; - } - - return 0; -} - -/// Abort transaction -/// Returns: 0 on success, -1 on error -export fn lithoglyph_nif_txn_abort(handle: *TxnHandle) c_int { - const status = lith_txn_abort(handle); - return if (status == .ok) 0 else -1; -} - -/// Apply operation to transaction -/// Input: transaction handle, operation CBOR buffer, operation length -/// Output: block_id (u64), has_provenance (bool/u32), provenance_hash (32 bytes) -/// Returns: 0 on success, -1 on error -export fn lithoglyph_nif_apply( - handle: *TxnHandle, - op_buffer: [*]const u8, - op_length: u32, - block_id_out: *u64, - has_provenance_out: *u32, - provenance_buffer: [*]u8, -) c_int { - // Call Lithoglyph apply - const result = lith_apply( - handle, - op_buffer, - op_length, - ); - - if (result.status != .ok) { - if (result.err_blob.toSlice()) |err_slice| { - std.log.err("Failed to apply operation: {s}", .{err_slice}); - } - return -1; - } - - // Extract block ID from result blob (CBOR-encoded) - if (result.result_blob.toSlice()) |result_data| { - // TODO: Parse CBOR to extract doc_id - // For now, use a placeholder block ID - block_id_out.* = 0x1; - _ = result_data; - } else { - block_id_out.* = 0; - } - - // Check if provenance is present - if (result.provenance_blob.toSlice()) |prov_data| { - has_provenance_out.* = 1; - // Copy provenance data (first 32 bytes as hash) - const copy_len = @min(prov_data.len, 32); - @memcpy(provenance_buffer[0..copy_len], prov_data[0..copy_len]); - if (copy_len < 32) { - @memset(provenance_buffer[copy_len..32], 0); - } - } else { - has_provenance_out.* = 0; - @memset(provenance_buffer[0..32], 0); - } - - return 0; -} - -/// Get database schema (CBOR-encoded) -/// Input: database handle, output buffer, max buffer size -/// Returns: actual data length, or 0 on error -export fn lithoglyph_nif_schema( - handle: *DbHandle, - buffer: [*]u8, - max_size: u32, -) u32 { - // TODO: Implement schema retrieval from Lithoglyph - // For now, return empty CBOR map {} - const empty_map = [_]u8{ - 0xA0, // CBOR: {} (empty map) - }; - - if (max_size < empty_map.len) { - return 0; // Buffer too small - } - - @memcpy(buffer[0..empty_map.len], &empty_map); - - _ = handle; - - return @intCast(empty_map.len); -} - -/// Get journal entries since timestamp -/// Input: database handle, since timestamp, output buffer, max buffer size -/// Returns: actual data length, or 0 on error -export fn lithoglyph_nif_journal( - handle: *DbHandle, - since: u64, - buffer: [*]u8, - max_size: u32, -) u32 { - // TODO: Implement journal retrieval from Lithoglyph - // For now, return empty CBOR array [] - const empty_array = [_]u8{ - 0x80, // CBOR: [] (empty array) - }; - - if (max_size < empty_array.len) { - return 0; // Buffer too small - } - - @memcpy(buffer[0..empty_array.len], &empty_array); - - _ = handle; - _ = since; - - return @intCast(empty_array.len); -} - -// ============================================================================ -// Erlang NIF initialization (for loading as Erlang NIF) -// ============================================================================ - -// NIF function table -const nif_funcs = [_]c.ErlNifFunc{ - .{ .name = "version", .arity = 0, .fptr = nif_version, .flags = 0 }, - .{ .name = "db_open", .arity = 1, .fptr = nif_db_open, .flags = 0 }, - .{ .name = "db_close", .arity = 1, .fptr = nif_db_close, .flags = 0 }, - .{ .name = "txn_begin", .arity = 2, .fptr = nif_txn_begin, .flags = 0 }, - .{ .name = "txn_commit", .arity = 1, .fptr = nif_txn_commit, .flags = 0 }, - .{ .name = "txn_abort", .arity = 1, .fptr = nif_txn_abort, .flags = 0 }, - .{ .name = "apply", .arity = 2, .fptr = nif_apply, .flags = 0 }, - .{ .name = "schema", .arity = 1, .fptr = nif_schema, .flags = 0 }, - .{ .name = "journal", .arity = 2, .fptr = nif_journal, .flags = 0 }, -}; - -// Erlang NIF wrapper functions -fn nif_version(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - - var major: u8 = undefined; - var minor: u8 = undefined; - var patch: u8 = undefined; - - lithoglyph_nif_version(&major, &minor, &patch); - - return c.enif_make_tuple3( - env, - c.enif_make_uint(env, major), - c.enif_make_uint(env, minor), - c.enif_make_uint(env, patch), - ); -} - -fn nif_db_open(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - - var path_binary: c.ErlNifBinary = undefined; - if (c.enif_inspect_binary(env, argv[0], &path_binary) == 0) { - return c.enif_make_badarg(env); - } - - // TODO: Convert binary to null-terminated string properly - // For now, assume path fits in a fixed buffer - var path_buf: [4096]u8 = undefined; - const path_len = @min(path_binary.size, path_buf.len - 1); - @memcpy(path_buf[0..path_len], path_binary.data[0..path_len]); - path_buf[path_len] = 0; // Null terminate - - const handle = lithoglyph_nif_db_open(@ptrCast(&path_buf)); - - if (handle) |h| { - // TODO: Create resource term for handle - // For now, return dummy term - _ = h; - return c.enif_make_atom(env, "ok"); - } else { - return c.enif_make_tuple2( - env, - c.enif_make_atom(env, "error"), - c.enif_make_atom(env, "failed_to_open"), - ); - } -} - -fn nif_db_close(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Extract resource handle from argv[0] - return c.enif_make_atom(env, "ok"); -} - -fn nif_txn_begin(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -fn nif_txn_commit(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -fn nif_txn_abort(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -fn nif_apply(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -fn nif_schema(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -fn nif_journal(env: ?*c.ErlNifEnv, argc: c_int, argv: [*c]const c.ERL_NIF_TERM) callconv(.c) c.ERL_NIF_TERM { - _ = argc; - _ = argv; - // TODO: Implement NIF wrapper - return c.enif_make_atom(env, "ok"); -} - -// NIF load function -fn nif_load(env: ?*c.ErlNifEnv, priv_data: [*c]?*anyopaque, load_info: c.ERL_NIF_TERM) callconv(.c) c_int { - _ = env; - _ = priv_data; - _ = load_info; - // TODO: Initialize resource types - return 0; -} - -// NIF unload function -fn nif_unload(env: ?*c.ErlNifEnv, priv_data: ?*anyopaque) callconv(.c) void { - _ = env; - _ = priv_data; - // Cleanup if needed -} - -// NIF entry point -export const lithoglyph_nif_entry = c.ErlNifEntry{ - .major = c.ERL_NIF_MAJOR_VERSION, - .minor = c.ERL_NIF_MINOR_VERSION, - .name = "lithoglyph_nif", - .num_of_funcs = nif_funcs.len, - .funcs = @ptrCast(@constCast(&nif_funcs)), - .load = nif_load, - .reload = null, - .upgrade = null, - .unload = nif_unload, - .vm_variant = "beam.vanilla", - .options = 0, - .sizeof_ErlNifResourceTypeInit = @sizeOf(c.ErlNifResourceTypeInit), -}; - -// Tests -test "version" { - var major: u8 = undefined; - var minor: u8 = undefined; - var patch: u8 = undefined; - - lithoglyph_nif_version(&major, &minor, &patch); - - try std.testing.expectEqual(@as(u8, 0), major); - try std.testing.expectEqual(@as(u8, 1), minor); - try std.testing.expectEqual(@as(u8, 0), patch); -} - -test "db lifecycle" { - const handle = lithoglyph_nif_db_open("/tmp/test.db"); - try std.testing.expect(handle != null); - - const result = lithoglyph_nif_db_close(handle.?); - try std.testing.expectEqual(@as(c_int, 0), result); -} diff --git a/lithoglyph/glyphbase/server/ffi/zig/test/integration_test.zig b/lithoglyph/glyphbase/server/ffi/zig/test/integration_test.zig deleted file mode 100644 index bc62d4a7..00000000 --- a/lithoglyph/glyphbase/server/ffi/zig/test/integration_test.zig +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Integration tests for Lith Zig FFI - -const std = @import("std"); -const testing = std.testing; - -// Import the main module -const lith = @import("../src/main.zig"); - -test "full database workflow" { - // Test version - var major: u8 = undefined; - var minor: u8 = undefined; - var patch: u8 = undefined; - lith.lith_nif_version(&major, &minor, &patch); - - try testing.expectEqual(@as(u8, 0), major); - try testing.expectEqual(@as(u8, 1), minor); - try testing.expectEqual(@as(u8, 0), patch); - - // Test database open/close - const db = lith.lith_nif_db_open("/tmp/integration_test.db"); - try testing.expect(db != null); - defer _ = lith.lith_nif_db_close(db.?); - - // Test transaction begin/commit - const txn = lith.lith_nif_txn_begin(db.?, 1); // ReadWrite - try testing.expect(txn != null); - - // Test apply operation - const dummy_op = [_]u8{ 0x00, 0x01, 0x02 }; // Dummy CBOR - var block_id: u64 = undefined; - var has_prov: u32 = undefined; - var prov_hash: [32]u8 = undefined; - - const apply_result = lith.lith_nif_apply( - txn.?, - &dummy_op, - dummy_op.len, - &block_id, - &has_prov, - &prov_hash, - ); - try testing.expectEqual(@as(c_int, 0), apply_result); - try testing.expect(block_id != 0); - try testing.expectEqual(@as(u32, 1), has_prov); - - // Test commit - const commit_result = lith.lith_nif_txn_commit(txn.?); - try testing.expectEqual(@as(c_int, 0), commit_result); -} - -test "schema retrieval" { - const db = lith.lith_nif_db_open("/tmp/schema_test.db"); - try testing.expect(db != null); - defer _ = lith.lith_nif_db_close(db.?); - - var buffer: [1024]u8 = undefined; - const len = lith.lith_nif_schema(db.?, &buffer, buffer.len); - - try testing.expect(len > 0); - try testing.expect(len <= buffer.len); -} - -test "journal retrieval" { - const db = lith.lith_nif_db_open("/tmp/journal_test.db"); - try testing.expect(db != null); - defer _ = lith.lith_nif_db_close(db.?); - - var buffer: [4096]u8 = undefined; - const since: u64 = 0; // From beginning - const len = lith.lith_nif_journal(db.?, since, &buffer, buffer.len); - - try testing.expect(len > 0); - try testing.expect(len <= buffer.len); -} - -test "transaction abort" { - const db = lith.lith_nif_db_open("/tmp/abort_test.db"); - try testing.expect(db != null); - defer _ = lith.lith_nif_db_close(db.?); - - const txn = lith.lith_nif_txn_begin(db.?, 1); // ReadWrite - try testing.expect(txn != null); - - const abort_result = lith.lith_nif_txn_abort(txn.?); - try testing.expectEqual(@as(c_int, 0), abort_result); -} - -test "read-only transaction" { - const db = lith.lith_nif_db_open("/tmp/readonly_test.db"); - try testing.expect(db != null); - defer _ = lith.lith_nif_db_close(db.?); - - const txn = lith.lith_nif_txn_begin(db.?, 0); // ReadOnly - try testing.expect(txn != null); - - const commit_result = lith.lith_nif_txn_commit(txn.?); - try testing.expectEqual(@as(c_int, 0), commit_result); -} diff --git a/lithoglyph/glyphbase/server/gleam.toml b/lithoglyph/glyphbase/server/gleam.toml deleted file mode 100644 index db3d46d1..00000000 --- a/lithoglyph/glyphbase/server/gleam.toml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# (PMPL-1.0-or-later preferred; MPL-2.0 required for Gleam ecosystem) - -name = "glyphbase_server" -version = "0.1.0" -description = "Glyphbase API server" -licences = ["MPL-2.0"] -repository = { type = "github", user = "hyperpolymath", repo = "glyphbase" } - -[dependencies] -gleam_stdlib = ">= 0.45.0" -gleam_http = ">= 3.7.0" -gleam_json = ">= 2.0.0" -gleam_erlang = ">= 0.34.0" -gleam_otp = ">= 0.16.0" -mist = ">= 4.0.0" -wisp = ">= 1.3.0" -envoy = ">= 1.1.0 and < 2.0.0" - -[dev-dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/lithoglyph/glyphbase/server/manifest.toml b/lithoglyph/glyphbase/server/manifest.toml deleted file mode 100644 index a7d89834..00000000 --- a/lithoglyph/glyphbase/server/manifest.toml +++ /dev/null @@ -1,39 +0,0 @@ -# This file was generated by Gleam -# You typically do not need to edit this file - -packages = [ - { name = "directories", version = "1.2.0", build_tools = ["gleam"], requirements = ["envoy", "gleam_stdlib", "platform", "simplifile"], otp_app = "directories", source = "hex", outer_checksum = "D13090CFCDF6759B87217E8DDD73A75903A700148A82C1D33799F333E249BF9E" }, - { name = "envoy", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envoy", source = "hex", outer_checksum = "850DA9D29D2E5987735872A2B5C81035146D7FE19EFC486129E44440D03FD832" }, - { name = "exception", version = "2.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "329D269D5C2A314F7364BD2711372B6F2C58FA6F39981572E5CA68624D291F8C" }, - { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, - { name = "gleam_crypto", version = "1.5.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "50774BAFFF1144E7872814C566C5D653D83A3EBF23ACC3156B757A1B6819086E" }, - { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, - { name = "gleam_http", version = "4.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "82EA6A717C842456188C190AFB372665EA56CE13D8559BF3B1DD9E40F619EE0C" }, - { name = "gleam_json", version = "3.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "44FDAA8847BE8FC48CA7A1C089706BD54BADCC4C45B237A992EDDF9F2CDB2836" }, - { name = "gleam_otp", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "BA6A294E295E428EC1562DC1C11EA7530DCB981E8359134BEABC8493B7B2258E" }, - { name = "gleam_stdlib", version = "0.68.1", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "F7FAEBD8EF260664E86A46C8DBA23508D1D11BB3BCC6EE1B89B3BC3E5C83FF1E" }, - { name = "gleam_yielder", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_yielder", source = "hex", outer_checksum = "8E4E4ECFA7982859F430C57F549200C7749823C106759F4A19A78AEA6687717A" }, - { name = "gleeunit", version = "1.9.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "DA9553CE58B67924B3C631F96FE3370C49EB6D6DC6B384EC4862CC4AAA718F3C" }, - { name = "glisten", version = "8.0.3", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib", "logging", "telemetry"], otp_app = "glisten", source = "hex", outer_checksum = "86B838196592D9EBDE7A1D2369AE3A51E568F7DD2D168706C463C42D17B95312" }, - { name = "gramps", version = "6.0.0", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_http", "gleam_stdlib"], otp_app = "gramps", source = "hex", outer_checksum = "8B7195978FBFD30B43DF791A8A272041B81E45D245314D7A41FC57237AA882A0" }, - { name = "houdini", version = "1.2.0", build_tools = ["gleam"], requirements = [], otp_app = "houdini", source = "hex", outer_checksum = "5DB1053F1AF828049C2B206D4403C18970ABEF5C18671CA3C2D2ED0DD64F6385" }, - { name = "hpack_erl", version = "0.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "hpack", source = "hex", outer_checksum = "D6137D7079169D8C485C6962DFE261AF5B9EF60FBC557344511C1E65E3D95FB0" }, - { name = "logging", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "1098FBF10B54B44C2C7FDF0B01C1253CAFACDACABEFB4B0D027803246753E06D" }, - { name = "marceau", version = "1.3.0", build_tools = ["gleam"], requirements = [], otp_app = "marceau", source = "hex", outer_checksum = "2D1C27504BEF45005F5DFB18591F8610FB4BFA91744878210BDC464412EC44E9" }, - { name = "mist", version = "5.0.4", build_tools = ["gleam"], requirements = ["exception", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "gleam_yielder", "glisten", "gramps", "hpack_erl", "logging"], otp_app = "mist", source = "hex", outer_checksum = "7CED4B2D81FD547ADB093D97B9928B9419A7F58B8562A30A6CC17A252B31AD05" }, - { name = "platform", version = "1.0.0", build_tools = ["gleam"], requirements = [], otp_app = "platform", source = "hex", outer_checksum = "8339420A95AD89AAC0F82F4C3DB8DD401041742D6C3F46132A8739F6AEB75391" }, - { name = "simplifile", version = "2.3.2", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "E049B4DACD4D206D87843BCF4C775A50AE0F50A52031A2FFB40C9ED07D6EC70A" }, - { name = "telemetry", version = "1.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "telemetry", source = "hex", outer_checksum = "7015FC8919DBE63764F4B4B87A95B7C0996BD539E0D499BE6EC9D7F3875B79E6" }, - { name = "wisp", version = "2.1.1", build_tools = ["gleam"], requirements = ["directories", "exception", "filepath", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "houdini", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "46E2E31DECD61A3748CF6CB317D9AC432BBC8D8A6E65655A9E787BDC69389DE0" }, -] - -[requirements] -envoy = { version = ">= 1.1.0 and < 2.0.0" } -gleam_erlang = { version = ">= 0.34.0" } -gleam_http = { version = ">= 3.7.0" } -gleam_json = { version = ">= 2.0.0" } -gleam_otp = { version = ">= 0.16.0" } -gleam_stdlib = { version = ">= 0.45.0" } -gleeunit = { version = ">= 1.0.0 and < 2.0.0" } -mist = { version = ">= 4.0.0" } -wisp = { version = ">= 1.3.0" } diff --git a/lithoglyph/glyphbase/server/native/Justfile b/lithoglyph/glyphbase/server/native/Justfile deleted file mode 100644 index 2d018cda..00000000 --- a/lithoglyph/glyphbase/server/native/Justfile +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# Justfile for Lith NIF C wrapper - -set shell := ["bash", "-euo", "pipefail", "-c"] - -default: - @just --list --unsorted - -all: - @erl_include="$$(erl -eval 'io:format("~s", [code:root_dir()])' -s init stop -noshell)/erts-$$(erl -eval 'io:format("~s", [erlang:system_info(version)])' -s init stop -noshell)/include" - @mkdir -p ../priv - ${CC:-cc} -fPIC -O2 -Wall -I"$$erl_include" lith_nif.c -o ../priv/lith_nif.so -shared -L../../../lith/ffi/zig/zig-out/lib -llith - @echo "Built ../priv/lith_nif.so" - -clean: - rm -f ../priv/lith_nif.so - rm -rf ../priv diff --git a/lithoglyph/glyphbase/server/native/build.zig b/lithoglyph/glyphbase/server/native/build.zig deleted file mode 100644 index f109ccde..00000000 --- a/lithoglyph/glyphbase/server/native/build.zig +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Build configuration for Lithoglyph NIF (Gleam/BEAM) -// -// Links the NIF against core-zig via Zig module import (same pattern as ffi/zig/). -// The NIF source imports core_bridge to access real Lith functions and types. - -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - // ================================================================ - // Core-zig module (the real storage engine) - // ================================================================ - - const core_bridge_mod = b.createModule(.{ - .root_source_file = b.path("../../../core-zig/src/bridge.zig"), - .target = target, - .optimize = optimize, - }); - - // ================================================================ - // Erlang NIF headers - // ================================================================ - - const erl_include = blk: { - if (std.process.getEnvVarOwned(b.allocator, "ERL_INCLUDE_PATH")) |path| { - break :blk path; - } else |_| {} - - const result = std.process.Child.run(.{ - .allocator = b.allocator, - .argv = &.{ "erl", "-noshell", "-eval", "io:format(\"~s\", [code:root_dir()])", "-s", "init", "stop" }, - }) catch { - @panic("Failed to find Erlang installation. Set ERL_INCLUDE_PATH."); - }; - - const root_dir = std.mem.trim(u8, result.stdout, &std.ascii.whitespace); - break :blk b.fmt("{s}/usr/include", .{root_dir}); - }; - - // ================================================================ - // NIF shared library (links core-zig via module import) - // ================================================================ - - const lib = b.addLibrary(.{ - .name = "lithoglyph_nif", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/lithoglyph_nif.zig"), - .target = target, - .optimize = optimize, - .imports = &.{ - .{ .name = "core_bridge", .module = core_bridge_mod }, - }, - }), - .linkage = .dynamic, - }); - - // Erlang NIF headers - lib.root_module.addIncludePath(.{ .cwd_relative = erl_include }); - - // libc for system calls - lib.linkLibC(); - - // Install to priv/ for Gleam/BEAM to find - const install = b.addInstallArtifact(lib, .{ - .dest_dir = .{ .override = .{ .custom = "../priv" } }, - }); - - b.getInstallStep().dependOn(&install.step); - - // ================================================================ - // Tests (with core-zig module available) - // ================================================================ - - const unit_tests = b.addTest(.{ - .name = "nif-tests", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/lithoglyph_nif.zig"), - .target = target, - .optimize = optimize, - .imports = &.{ - .{ .name = "core_bridge", .module = core_bridge_mod }, - }, - }), - }); - - unit_tests.root_module.addIncludePath(.{ .cwd_relative = erl_include }); - - const run_unit_tests = b.addRunArtifact(unit_tests); - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_unit_tests.step); -} diff --git a/lithoglyph/glyphbase/server/native/lith_nif.c b/lithoglyph/glyphbase/server/native/lith_nif.c deleted file mode 100644 index a02a1ed3..00000000 --- a/lithoglyph/glyphbase/server/native/lith_nif.c +++ /dev/null @@ -1,352 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2025 Jonathan D.A. Jewell (@hyperpolymath) -// -// lith_nif.c - Erlang NIF wrapper for Lith Zig FFI -// -// This bridges Gleam (BEAM) to Lith's liblith.so - -#include -#include -#include - -// Forward declarations for Lith FFI functions -// These will be linked from liblith.so - -extern int32_t lith_init(void); -extern void lith_cleanup(void); -extern int32_t lith_open(const char* path, uint64_t path_len, void** db_out); -extern int32_t lith_close(void* db); -extern int32_t lith_create(const char* path, uint64_t path_len, uint64_t block_count, void** db_out); - -extern int32_t lith_txn_begin(void* db, void** txn_out); -extern int32_t lith_txn_commit(void* txn); -extern int32_t lith_txn_rollback(void* txn); - -extern int32_t lith_query_execute( - void* db, - const char* query_str, - uint64_t query_len, - const char* provenance_json, - uint64_t provenance_len, - void** cursor_out -); - -extern int32_t lith_cursor_next( - void* cursor, - char* document_json_out, - uint64_t buffer_len, - uint64_t* written_out -); - -extern void lith_cursor_close(void* cursor); - -// Status codes (matches Lith ABI) -#define STATUS_OK 0 -#define STATUS_INVALID_ARG 1 -#define STATUS_NOT_FOUND 2 -#define STATUS_PERMISSION_DENIED 3 -#define STATUS_ALREADY_EXISTS 4 -#define STATUS_CONSTRAINT_VIOLATION 5 -#define STATUS_TYPE_MISMATCH 6 -#define STATUS_OUT_OF_MEMORY 7 -#define STATUS_IO_ERROR 8 -#define STATUS_CORRUPTION 9 -#define STATUS_CONFLICT 10 -#define STATUS_INTERNAL_ERROR 11 - -// Resource types for Erlang resource management -static ErlNifResourceType *LITH_DB_RESOURCE; -static ErlNifResourceType *LITH_TXN_RESOURCE; -static ErlNifResourceType *LITH_CURSOR_RESOURCE; - -// Resource wrapper structures -typedef struct { - void* handle; -} LithDbResource; - -typedef struct { - void* handle; -} LithTxnResource; - -typedef struct { - void* handle; -} LithCursorResource; - -// Resource destructor for database -static void lith_db_resource_dtor(ErlNifEnv* env, void* obj) { - LithDbResource* res = (LithDbResource*)obj; - if (res->handle != NULL) { - lith_close(res->handle); - res->handle = NULL; - } -} - -// Resource destructor for transaction -static void lith_txn_resource_dtor(ErlNifEnv* env, void* obj) { - LithTxnResource* res = (LithTxnResource*)obj; - if (res->handle != NULL) { - lith_txn_rollback(res->handle); // Auto-rollback on GC - res->handle = NULL; - } -} - -// Resource destructor for cursor -static void lith_cursor_resource_dtor(ErlNifEnv* env, void* obj) { - LithCursorResource* res = (LithCursorResource*)obj; - if (res->handle != NULL) { - lith_cursor_close(res->handle); - res->handle = NULL; - } -} - -// Helper: Convert status code to Erlang atom -// Convert status code to Gleam LithError atom -static ERL_NIF_TERM status_to_error_atom(ErlNifEnv* env, int32_t status) { - switch (status) { - case STATUS_INVALID_ARG: return enif_make_atom(env, "InvalidArg"); - case STATUS_NOT_FOUND: return enif_make_atom(env, "NotFound"); - case STATUS_PERMISSION_DENIED: return enif_make_atom(env, "PermissionDenied"); - case STATUS_ALREADY_EXISTS: return enif_make_atom(env, "AlreadyExists"); - case STATUS_CONSTRAINT_VIOLATION: return enif_make_atom(env, "ConstraintViolation"); - case STATUS_TYPE_MISMATCH: return enif_make_atom(env, "TypeMismatch"); - case STATUS_OUT_OF_MEMORY: return enif_make_atom(env, "OutOfMemory"); - case STATUS_IO_ERROR: return enif_make_atom(env, "IoError"); - case STATUS_CORRUPTION: return enif_make_atom(env, "Corruption"); - case STATUS_CONFLICT: return enif_make_atom(env, "Conflict"); - case STATUS_INTERNAL_ERROR: - default: return enif_make_atom(env, "InternalError"); - } -} - -// Convert status code to Gleam Result(Nil, LithError) -static ERL_NIF_TERM status_to_result(ErlNifEnv* env, int32_t status) { - if (status == STATUS_OK) { - // Return {ok, nil} for Gleam Result type - return enif_make_tuple2(env, - enif_make_atom(env, "ok"), - enif_make_atom(env, "nil")); - } else { - // Return {error, ErrorAtom} for Gleam Result type - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// NIF Functions -//////////////////////////////////////////////////////////////////////////////// - -// Initialize Lith -static ERL_NIF_TERM nif_lith_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - int32_t status = lith_init(); - return status_to_result(env, status); -} - -// Open database: open(Path) -> {ok, DbRef} | {error, Reason} -static ERL_NIF_TERM nif_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary path_bin; - - if (!enif_inspect_binary(env, argv[0], &path_bin)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - LithDbResource* db_res = enif_alloc_resource(LITH_DB_RESOURCE, sizeof(LithDbResource)); - db_res->handle = NULL; - - int32_t status = lith_open((const char*)path_bin.data, path_bin.size, &db_res->handle); - - if (status == STATUS_OK && db_res->handle != NULL) { - ERL_NIF_TERM db_term = enif_make_resource(env, db_res); - enif_release_resource(db_res); - return enif_make_tuple2(env, enif_make_atom(env, "ok"), db_term); - } else { - enif_release_resource(db_res); - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -// Create database: create(Path, BlockCount) -> {ok, DbRef} | {error, Reason} -static ERL_NIF_TERM nif_create(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary path_bin; - uint64_t block_count; - - if (!enif_inspect_binary(env, argv[0], &path_bin) || - !enif_get_uint64(env, argv[1], &block_count)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - LithDbResource* db_res = enif_alloc_resource(LITH_DB_RESOURCE, sizeof(LithDbResource)); - db_res->handle = NULL; - - int32_t status = lith_create((const char*)path_bin.data, path_bin.size, block_count, &db_res->handle); - - if (status == STATUS_OK && db_res->handle != NULL) { - ERL_NIF_TERM db_term = enif_make_resource(env, db_res); - enif_release_resource(db_res); - return enif_make_tuple2(env, enif_make_atom(env, "ok"), db_term); - } else { - enif_release_resource(db_res); - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -// Begin transaction: txn_begin(DbRef) -> {ok, TxnRef} | {error, Reason} -static ERL_NIF_TERM nif_txn_begin(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - LithDbResource* db_res; - - if (!enif_get_resource(env, argv[0], LITH_DB_RESOURCE, (void**)&db_res)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - LithTxnResource* txn_res = enif_alloc_resource(LITH_TXN_RESOURCE, sizeof(LithTxnResource)); - txn_res->handle = NULL; - - int32_t status = lith_txn_begin(db_res->handle, &txn_res->handle); - - if (status == STATUS_OK && txn_res->handle != NULL) { - ERL_NIF_TERM txn_term = enif_make_resource(env, txn_res); - enif_release_resource(txn_res); - return enif_make_tuple2(env, enif_make_atom(env, "ok"), txn_term); - } else { - enif_release_resource(txn_res); - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -// Commit transaction: txn_commit(TxnRef) -> ok | {error, Reason} -static ERL_NIF_TERM nif_txn_commit(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - LithTxnResource* txn_res; - - if (!enif_get_resource(env, argv[0], LITH_TXN_RESOURCE, (void**)&txn_res)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - int32_t status = lith_txn_commit(txn_res->handle); - - if (status == STATUS_OK) { - return enif_make_atom(env, "ok"); - } else { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -// Execute query: query_execute(DbRef, QueryStr, ProvenanceJson) -> {ok, CursorRef} | {error, Reason} -static ERL_NIF_TERM nif_query_execute(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - LithDbResource* db_res; - ErlNifBinary query_bin, prov_bin; - - if (!enif_get_resource(env, argv[0], LITH_DB_RESOURCE, (void**)&db_res) || - !enif_inspect_binary(env, argv[1], &query_bin) || - !enif_inspect_binary(env, argv[2], &prov_bin)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - LithCursorResource* cursor_res = enif_alloc_resource(LITH_CURSOR_RESOURCE, sizeof(LithCursorResource)); - cursor_res->handle = NULL; - - int32_t status = lith_query_execute( - db_res->handle, - (const char*)query_bin.data, - query_bin.size, - (const char*)prov_bin.data, - prov_bin.size, - &cursor_res->handle - ); - - if (status == STATUS_OK && cursor_res->handle != NULL) { - ERL_NIF_TERM cursor_term = enif_make_resource(env, cursor_res); - enif_release_resource(cursor_res); - return enif_make_tuple2(env, enif_make_atom(env, "ok"), cursor_term); - } else { - enif_release_resource(cursor_res); - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -// Fetch next from cursor: cursor_next(CursorRef) -> {ok, JsonDoc} | done | {error, Reason} -static ERL_NIF_TERM nif_cursor_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - LithCursorResource* cursor_res; - - if (!enif_get_resource(env, argv[0], LITH_CURSOR_RESOURCE, (void**)&cursor_res)) { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); - } - - // Allocate buffer for JSON result (64KB should be enough for most documents) - char buffer[65536]; - uint64_t written = 0; - - int32_t status = lith_cursor_next(cursor_res->handle, buffer, sizeof(buffer), &written); - - if (status == STATUS_OK) { - ERL_NIF_TERM json_bin; - unsigned char* bin_data = enif_make_new_binary(env, written, &json_bin); - memcpy(bin_data, buffer, written); - return enif_make_tuple2(env, enif_make_atom(env, "ok"), json_bin); - } else if (status == STATUS_NOT_FOUND) { - return enif_make_atom(env, "done"); - } else { - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - status_to_error_atom(env, status)); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// NIF Module Setup -//////////////////////////////////////////////////////////////////////////////// - -static ErlNifFunc nif_funcs[] = { - {"init", 0, nif_lith_init}, - {"open", 1, nif_open}, - {"create", 2, nif_create}, - {"txn_begin", 1, nif_txn_begin}, - {"txn_commit", 1, nif_txn_commit}, - {"query_execute", 3, nif_query_execute}, - {"cursor_next", 1, nif_cursor_next} -}; - -static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - // Create resource types - LITH_DB_RESOURCE = enif_open_resource_type( - env, NULL, "lith_db", lith_db_resource_dtor, - ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - - LITH_TXN_RESOURCE = enif_open_resource_type( - env, NULL, "lith_txn", lith_txn_resource_dtor, - ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - - LITH_CURSOR_RESOURCE = enif_open_resource_type( - env, NULL, "lith_cursor", lith_cursor_resource_dtor, - ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - - if (LITH_DB_RESOURCE == NULL || LITH_TXN_RESOURCE == NULL || LITH_CURSOR_RESOURCE == NULL) { - return -1; - } - - return 0; -} - -ERL_NIF_INIT(lith_nif, nif_funcs, load, NULL, NULL, NULL) diff --git a/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.erl b/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.erl deleted file mode 100644 index 5ff26df6..00000000 --- a/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.erl +++ /dev/null @@ -1,107 +0,0 @@ -%% SPDX-License-Identifier: MPL-2.0 -%% Lith NIF - Erlang interface to Lith -%% -%% This module loads the Zig NIF and provides Erlang functions -%% to interact with Lith. - --module(lith_nif). --export([ - version/0, - db_open/1, - db_close/1, - txn_begin/2, - txn_commit/1, - txn_abort/1, - apply/2, - schema/1, - journal/2 -]). - --on_load(init/0). - --define(NIF_NOT_LOADED, erlang:nif_error(nif_not_loaded)). - -%% @doc Initialize the NIF -init() -> - PrivDir = case code:priv_dir(formbase_server) of - {error, _} -> - %% Fallback for development - case code:which(?MODULE) of - Filename when is_list(Filename) -> - filename:join([filename:dirname(Filename), "..", "priv"]); - _ -> - "priv" - end; - Dir -> - Dir - end, - SoPath = filename:join(PrivDir, "lith_nif"), - erlang:load_nif(SoPath, 0). - -%% @doc Get Lith version as {Major, Minor, Patch} --spec version() -> {non_neg_integer(), non_neg_integer(), non_neg_integer()}. -version() -> - ?NIF_NOT_LOADED. - -%% @doc Open a Lith database -%% @param Path Binary path to the database directory -%% @returns {ok, DbRef} | {error, Reason} --spec db_open(binary()) -> {ok, reference()} | {error, atom()}. -db_open(_Path) -> - ?NIF_NOT_LOADED. - -%% @doc Close a Lith database -%% @param DbRef Database reference from db_open/1 -%% @returns ok | {error, Reason} --spec db_close(reference()) -> ok | {error, atom()}. -db_close(_DbRef) -> - ?NIF_NOT_LOADED. - -%% @doc Begin a transaction -%% @param DbRef Database reference -%% @param Mode Transaction mode: read_only | read_write -%% @returns {ok, TxnRef} | {error, Reason} --spec txn_begin(reference(), read_only | read_write) -> {ok, reference()} | {error, atom()}. -txn_begin(_DbRef, _Mode) -> - ?NIF_NOT_LOADED. - -%% @doc Commit a transaction -%% @param TxnRef Transaction reference -%% @returns ok | {error, Reason} --spec txn_commit(reference()) -> ok | {error, atom()}. -txn_commit(_TxnRef) -> - ?NIF_NOT_LOADED. - -%% @doc Abort a transaction -%% @param TxnRef Transaction reference -%% @returns ok --spec txn_abort(reference()) -> ok. -txn_abort(_TxnRef) -> - ?NIF_NOT_LOADED. - -%% @doc Apply an operation within a transaction -%% @param TxnRef Transaction reference -%% @param OpCbor CBOR-encoded operation -%% @returns {ok, ResultCbor} | {ok, ResultCbor, ProvenanceCbor} | {error, Reason} --spec apply(reference(), binary()) -> - {ok, binary()} | - {ok, binary(), binary()} | - {error, atom()} | - {error, atom(), binary()}. -apply(_TxnRef, _OpCbor) -> - ?NIF_NOT_LOADED. - -%% @doc Get database schema -%% @param DbRef Database reference -%% @returns {ok, SchemaCbor} | {error, Reason} --spec schema(reference()) -> {ok, binary()} | {error, atom()}. -schema(_DbRef) -> - ?NIF_NOT_LOADED. - -%% @doc Get journal entries since a sequence number -%% @param DbRef Database reference -%% @param Since Sequence number to start from -%% @returns {ok, JournalCbor} | {error, Reason} --spec journal(reference(), non_neg_integer()) -> {ok, binary()} | {error, atom()}. -journal(_DbRef, _Since) -> - ?NIF_NOT_LOADED. diff --git a/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.zig b/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.zig deleted file mode 100644 index f8bed886..00000000 --- a/lithoglyph/glyphbase/server/native/src/lithoglyph_nif.zig +++ /dev/null @@ -1,462 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -// -// Lithoglyph NIF - Erlang/BEAM Native Implemented Functions -// -// Bridges Lith's Zig storage engine to the Erlang runtime via NIFs. -// Uses core_bridge module import (not extern linking) for type safety. -// All data is passed as CBOR-encoded binaries. - -const std = @import("std"); -const core = @import("core_bridge"); -const erl_nif = @cImport({ - @cInclude("erl_nif.h"); -}); - -// Re-export core types for clarity -const LgBlob = core.LgBlob; -const LgStatus = core.LgStatus; -const LgResult = core.LgResult; -const LgTxnMode = core.LgTxnMode; -const LgRenderOpts = core.LgRenderOpts; - -// NIF function signature type (matches erl_nif.h expectations) -const NifEnv = ?*erl_nif.ErlNifEnv; -const NifTerm = erl_nif.ERL_NIF_TERM; -const NifArgs = [*c]const NifTerm; - -// ============================================================ -// Resource Types for BEAM -// ============================================================ - -var db_resource_type: ?*erl_nif.ErlNifResourceType = null; -var txn_resource_type: ?*erl_nif.ErlNifResourceType = null; - -const DbResource = struct { - handle: ?*anyopaque, -}; - -const TxnResource = struct { - handle: ?*anyopaque, - db: *DbResource, -}; - -fn db_resource_dtor(_: NifEnv, obj: ?*anyopaque) callconv(.c) void { - const res: *DbResource = @ptrCast(@alignCast(obj orelse return)); - if (res.handle) |h| { - // SAFETY: handle was created by lith_db_open which returns *LgDb as *anyopaque - _ = core.lith_db_close(@ptrCast(h)); - res.handle = null; - } -} - -fn txn_resource_dtor(_: NifEnv, obj: ?*anyopaque) callconv(.c) void { - const res: *TxnResource = @ptrCast(@alignCast(obj orelse return)); - if (res.handle) |h| { - // SAFETY: handle was created by lith_txn_begin which returns *LgTxn as *anyopaque - _ = core.lith_txn_abort(@ptrCast(h)); - res.handle = null; - } -} - -// ============================================================ -// NIF Helper Functions -// ============================================================ - -fn make_atom(env: NifEnv, name: []const u8) NifTerm { - var atom: erl_nif.ERL_NIF_TERM = undefined; - if (erl_nif.enif_make_existing_atom_len(env, name.ptr, name.len, &atom, erl_nif.ERL_NIF_LATIN1) != 0) { - return atom; - } - return erl_nif.enif_make_atom_len(env, name.ptr, name.len); -} - -fn make_ok(env: NifEnv, term: NifTerm) NifTerm { - return erl_nif.enif_make_tuple2(env, make_atom(env, "ok"), term); -} - -fn make_error(env: NifEnv, reason: []const u8) NifTerm { - return erl_nif.enif_make_tuple2(env, make_atom(env, "error"), make_atom(env, reason)); -} - -fn make_error_with_message(env: NifEnv, reason: []const u8, msg: []const u8) NifTerm { - var bin: erl_nif.ErlNifBinary = undefined; - if (erl_nif.enif_alloc_binary(msg.len, &bin) == 0) { - return make_error(env, reason); - } - @memcpy(bin.data[0..msg.len], msg); - const msg_term = erl_nif.enif_make_binary(env, &bin); - return erl_nif.enif_make_tuple3(env, make_atom(env, "error"), make_atom(env, reason), msg_term); -} - -fn blob_to_binary(env: NifEnv, blob: LgBlob) ?erl_nif.ERL_NIF_TERM { - if (blob.ptr) |data| { - var bin: erl_nif.ErlNifBinary = undefined; - if (erl_nif.enif_alloc_binary(blob.len, &bin) == 0) { - return null; - } - @memcpy(bin.data[0..blob.len], data[0..blob.len]); - return erl_nif.enif_make_binary(env, &bin); - } - var empty_bin = erl_nif.ErlNifBinary{ .size = 0, .data = null }; - return erl_nif.enif_make_binary(env, &empty_bin); -} - -fn status_to_atom(status: LgStatus) []const u8 { - return switch (status) { - .ok => "ok", - .err_internal => "internal_error", - .err_not_found => "not_found", - .err_invalid_argument => "invalid_argument", - .err_out_of_memory => "out_of_memory", - .err_not_implemented => "not_implemented", - .err_txn_not_active => "txn_not_active", - .err_txn_already_committed => "txn_already_committed", - }; -} - -// ============================================================ -// NIF Functions -// ============================================================ - -/// Get Lith version -fn nif_version(env: NifEnv, _: c_int, _: NifArgs) callconv(.c) NifTerm { - const version = core.lith_version(); - const major = version / 10000; - const minor = (version % 10000) / 100; - const patch = version % 100; - - return erl_nif.enif_make_tuple3( - env, - erl_nif.enif_make_uint(env, major), - erl_nif.enif_make_uint(env, minor), - erl_nif.enif_make_uint(env, patch), - ); -} - -/// Open a Lith database -fn nif_db_open(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 1) return make_error(env, "badarg"); - - var path_bin: erl_nif.ErlNifBinary = undefined; - if (erl_nif.enif_inspect_binary(env, argv[0], &path_bin) == 0) { - return make_error(env, "badarg"); - } - - // Allocate resource - const res: *DbResource = @ptrCast(@alignCast( - erl_nif.enif_alloc_resource(db_resource_type, @sizeOf(DbResource)) orelse - return make_error(env, "alloc_failed"), - )); - - var err_blob: LgBlob = LgBlob.empty(); - const status = core.lith_db_open( - path_bin.data, - path_bin.size, - null, - 0, - @ptrCast(&res.handle), - &err_blob, - ); - - if (status != .ok) { - erl_nif.enif_release_resource(res); - if (err_blob.ptr) |_| { - const msg = err_blob.ptr.?[0..err_blob.len]; - var blob_copy = err_blob; - defer core.lith_blob_free(&blob_copy); - return make_error_with_message(env, status_to_atom(status), msg); - } - return make_error(env, status_to_atom(status)); - } - - const term = erl_nif.enif_make_resource(env, res); - erl_nif.enif_release_resource(res); - return make_ok(env, term); -} - -/// Close a Lith database -fn nif_db_close(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 1) return make_error(env, "badarg"); - - var res: *DbResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], db_resource_type, @ptrCast(&res)) == 0) { - return make_error(env, "badarg"); - } - - if (res.handle) |h| { - // SAFETY: handle was created by lith_db_open - const status = core.lith_db_close(@ptrCast(h)); - res.handle = null; - if (status != .ok) { - return make_error(env, status_to_atom(status)); - } - } - - return make_atom(env, "ok"); -} - -/// Begin a transaction -fn nif_txn_begin(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 2) return make_error(env, "badarg"); - - var db_res: *DbResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], db_resource_type, @ptrCast(&db_res)) == 0) { - return make_error(env, "badarg"); - } - - if (db_res.handle == null) { - return make_error(env, "db_closed"); - } - - // Parse mode atom - var mode_buf: [32]u8 = undefined; - const mode_len = erl_nif.enif_get_atom(env, argv[1], &mode_buf, mode_buf.len, erl_nif.ERL_NIF_LATIN1); - if (mode_len == 0) return make_error(env, "badarg"); - - const mode_str = mode_buf[0 .. @as(usize, @intCast(mode_len)) - 1]; - const mode: LgTxnMode = if (std.mem.eql(u8, mode_str, "read_only")) - .read_only - else if (std.mem.eql(u8, mode_str, "read_write")) - .read_write - else - return make_error(env, "invalid_mode"); - - // Allocate transaction resource - const txn_res: *TxnResource = @ptrCast(@alignCast( - erl_nif.enif_alloc_resource(txn_resource_type, @sizeOf(TxnResource)) orelse - return make_error(env, "alloc_failed"), - )); - - txn_res.db = db_res; - - var err_blob: LgBlob = LgBlob.empty(); - // SAFETY: db_res.handle was created by lith_db_open; txn_res.handle receives *LgTxn - const status = core.lith_txn_begin(@ptrCast(db_res.handle), mode, @ptrCast(&txn_res.handle), &err_blob); - - if (status != .ok) { - erl_nif.enif_release_resource(txn_res); - return make_error(env, status_to_atom(status)); - } - - const term = erl_nif.enif_make_resource(env, txn_res); - erl_nif.enif_release_resource(txn_res); - return make_ok(env, term); -} - -/// Commit a transaction -fn nif_txn_commit(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 1) return make_error(env, "badarg"); - - var txn_res: *TxnResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], txn_resource_type, @ptrCast(&txn_res)) == 0) { - return make_error(env, "badarg"); - } - - if (txn_res.handle == null) { - return make_error(env, "txn_closed"); - } - - var err_blob: LgBlob = LgBlob.empty(); - // SAFETY: txn_res.handle was created by lith_txn_begin - const status = core.lith_txn_commit(@ptrCast(txn_res.handle), &err_blob); - txn_res.handle = null; - - if (status != .ok) { - return make_error(env, status_to_atom(status)); - } - - return make_atom(env, "ok"); -} - -/// Abort a transaction -fn nif_txn_abort(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 1) return make_error(env, "badarg"); - - var txn_res: *TxnResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], txn_resource_type, @ptrCast(&txn_res)) == 0) { - return make_error(env, "badarg"); - } - - if (txn_res.handle) |h| { - // SAFETY: handle was created by lith_txn_begin - _ = core.lith_txn_abort(@ptrCast(h)); - txn_res.handle = null; - } - - return make_atom(env, "ok"); -} - -/// Apply an operation (CBOR-encoded) -fn nif_apply(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 2) return make_error(env, "badarg"); - - var txn_res: *TxnResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], txn_resource_type, @ptrCast(&txn_res)) == 0) { - return make_error(env, "badarg"); - } - - if (txn_res.handle == null) { - return make_error(env, "txn_closed"); - } - - var op_bin: erl_nif.ErlNifBinary = undefined; - if (erl_nif.enif_inspect_binary(env, argv[1], &op_bin) == 0) { - return make_error(env, "badarg"); - } - - // SAFETY: txn_res.handle was created by lith_txn_begin - const result = core.lith_apply(@ptrCast(txn_res.handle), op_bin.data, op_bin.size); - - if (result.status != .ok) { - if (result.error_blob.ptr) |_| { - if (blob_to_binary(env, result.error_blob)) |err_term| { - return erl_nif.enif_make_tuple3( - env, - make_atom(env, "error"), - make_atom(env, status_to_atom(result.status)), - err_term, - ); - } - } - return make_error(env, status_to_atom(result.status)); - } - - // Build success response with result and provenance - const result_term = blob_to_binary(env, result.data) orelse - return make_error(env, "encoding_failed"); - - if (result.provenance.ptr != null) { - const prov_term = blob_to_binary(env, result.provenance) orelse - return make_ok(env, result_term); - - return erl_nif.enif_make_tuple3( - env, - make_atom(env, "ok"), - result_term, - prov_term, - ); - } - - return make_ok(env, result_term); -} - -/// Get schema information -fn nif_schema(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 1) return make_error(env, "badarg"); - - var db_res: *DbResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], db_resource_type, @ptrCast(&db_res)) == 0) { - return make_error(env, "badarg"); - } - - if (db_res.handle == null) { - return make_error(env, "db_closed"); - } - - var schema_blob: LgBlob = LgBlob.empty(); - var err_blob: LgBlob = LgBlob.empty(); - // SAFETY: db_res.handle was created by lith_db_open - const status = core.lith_introspect_schema(@ptrCast(db_res.handle), &schema_blob, &err_blob); - - if (status != .ok) { - return make_error(env, status_to_atom(status)); - } - - const schema_term = blob_to_binary(env, schema_blob) orelse - return make_error(env, "encoding_failed"); - - return make_ok(env, schema_term); -} - -/// Get journal entries since a sequence number -fn nif_journal(env: NifEnv, argc: c_int, argv: NifArgs) callconv(.c) NifTerm { - if (argc != 2) return make_error(env, "badarg"); - - var db_res: *DbResource = undefined; - if (erl_nif.enif_get_resource(env, argv[0], db_resource_type, @ptrCast(&db_res)) == 0) { - return make_error(env, "badarg"); - } - - if (db_res.handle == null) { - return make_error(env, "db_closed"); - } - - var since: c_ulong = undefined; - if (erl_nif.enif_get_ulong(env, argv[1], &since) == 0) { - return make_error(env, "badarg"); - } - - var journal_blob: LgBlob = LgBlob.empty(); - var err_blob: LgBlob = LgBlob.empty(); - const opts = LgRenderOpts{ .format = 0, .include_metadata = true }; - // SAFETY: db_res.handle was created by lith_db_open - const status = core.lith_render_journal(@ptrCast(db_res.handle), since, opts, &journal_blob, &err_blob); - - if (status != .ok) { - return make_error(env, status_to_atom(status)); - } - - const journal_term = blob_to_binary(env, journal_blob) orelse - return make_error(env, "encoding_failed"); - - return make_ok(env, journal_term); -} - -// ============================================================ -// NIF Table and Initialization -// ============================================================ - -var nif_funcs = [_]erl_nif.ErlNifFunc{ - .{ .name = "version", .arity = 0, .fptr = nif_version, .flags = 0 }, - .{ .name = "db_open", .arity = 1, .fptr = nif_db_open, .flags = 0 }, - .{ .name = "db_close", .arity = 1, .fptr = nif_db_close, .flags = 0 }, - .{ .name = "txn_begin", .arity = 2, .fptr = nif_txn_begin, .flags = 0 }, - .{ .name = "txn_commit", .arity = 1, .fptr = nif_txn_commit, .flags = 0 }, - .{ .name = "txn_abort", .arity = 1, .fptr = nif_txn_abort, .flags = 0 }, - .{ .name = "apply", .arity = 2, .fptr = nif_apply, .flags = 0 }, - .{ .name = "schema", .arity = 1, .fptr = nif_schema, .flags = 0 }, - .{ .name = "journal", .arity = 2, .fptr = nif_journal, .flags = 0 }, -}; - -fn load(env: NifEnv, _: [*c]?*anyopaque, _: erl_nif.ERL_NIF_TERM) callconv(.c) c_int { - db_resource_type = erl_nif.enif_open_resource_type( - env, - null, - "lith_db", - db_resource_dtor, - erl_nif.ERL_NIF_RT_CREATE, - null, - ); - - txn_resource_type = erl_nif.enif_open_resource_type( - env, - null, - "lith_txn", - txn_resource_dtor, - erl_nif.ERL_NIF_RT_CREATE, - null, - ); - - if (db_resource_type == null or txn_resource_type == null) { - return -1; - } - - return 0; -} - -pub export const lith_nif_init = erl_nif.ErlNifEntry{ - .major = erl_nif.ERL_NIF_MAJOR_VERSION, - .minor = erl_nif.ERL_NIF_MINOR_VERSION, - .name = "lith_nif", - .num_of_funcs = nif_funcs.len, - .funcs = &nif_funcs, - .load = load, - .reload = null, - .upgrade = null, - .unload = null, - .vm_variant = "beam.vanilla", - .options = 0, - .sizeof_ErlNifResourceTypeInit = @sizeOf(erl_nif.ErlNifResourceTypeInit), - .min_erts = "erts-13.0", -}; diff --git a/lithoglyph/glyphbase/server/native_rust/Cargo.toml b/lithoglyph/glyphbase/server/native_rust/Cargo.toml deleted file mode 100644 index abd94fae..00000000 --- a/lithoglyph/glyphbase/server/native_rust/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -[package] -name = "lith_nif" -version = "1.0.0" -edition = "2021" -authors = ["Jonathan D.A. Jewell "] - -[lib] -name = "lith_nif" -crate-type = ["cdylib"] - -[dependencies] -rustler = "0.35" diff --git a/lithoglyph/glyphbase/server/native_rust/src/lib.rs b/lithoglyph/glyphbase/server/native_rust/src/lib.rs deleted file mode 100644 index 0f8b55ef..00000000 --- a/lithoglyph/glyphbase/server/native_rust/src/lib.rs +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph BEAM NIF - Rust/Rustler implementation -// -// This NIF connects BEAM (Erlang/Gleam/Elixir) to Lithoglyph via the Lithoglyph C ABI. -// Uses CBOR-encoded binaries for efficient data transfer. - -#![forbid(unsafe_code)] -use rustler::{Encoder, Env, Error, ResourceArc, Term}; - -mod atoms { - rustler::atoms! { - ok, - error, - init_failed, - invalid_handle, - parse_failed, - validation_failed, - persist_failed, - alloc_failed, - } -} - -// Database handle wrapper -struct DbHandle { - path: String, - // M10 PoC: stub implementation, no actual gforth handle yet - _dummy: u64, -} - -// Transaction handle wrapper -struct TxnHandle { - db: ResourceArc, - mode: TxnMode, -} - -#[derive(Clone, Copy)] -enum TxnMode { - ReadOnly, - ReadWrite, -} - -// NIF functions -rustler::init!( - "lithoglyph_nif", - [ - version, - db_open, - db_close, - txn_begin, - txn_commit, - txn_abort, - apply, - schema, - journal - ], - load = load -); - -fn load(env: Env, _info: Term) -> bool { - rustler::resource!(DbHandle, env); - rustler::resource!(TxnHandle, env); - true -} - -/// Get Lithoglyph version -#[rustler::nif] -fn version() -> (i32, i32, i32) { - (1, 0, 0) // v1.0.0 for M10 -} - -/// Open a Lithoglyph database -#[rustler::nif] -fn db_open(path: String) -> ResourceArc { - let db = DbHandle { - path, - _dummy: 0xDEADBEEF, - }; - - ResourceArc::new(db) -} - -/// Close a Lithoglyph database -#[rustler::nif] -fn db_close(_db: ResourceArc) -> rustler::Atom { - // M10 PoC: no cleanup needed for stub - atoms::ok() -} - -/// Begin a transaction -#[rustler::nif] -fn txn_begin( - db: ResourceArc, - mode: String, -) -> Result, rustler::Atom> { - let txn_mode = match mode.as_str() { - "read_only" => TxnMode::ReadOnly, - "read_write" => TxnMode::ReadWrite, - _ => return Err(atoms::invalid_handle()), - }; - - let txn = TxnHandle { - db, - mode: txn_mode, - }; - - Ok(ResourceArc::new(txn)) -} - -/// Commit a transaction -#[rustler::nif] -fn txn_commit(_txn: ResourceArc) -> rustler::Atom { - // M10 PoC: no actual commit needed for stub - atoms::ok() -} - -/// Abort a transaction -#[rustler::nif] -fn txn_abort(_txn: ResourceArc) -> rustler::Atom { - atoms::ok() -} - -/// Apply an operation within a transaction -#[rustler::nif] -fn apply( - _txn: ResourceArc, - op_cbor: rustler::Binary, -) -> Result, rustler::Atom> { - // M10 PoC: Validate CBOR is a map, then return dummy block ID - - if op_cbor.is_empty() || op_cbor.len() > 1_048_576 { - return Err(atoms::parse_failed()); - } - - // Check first byte is CBOR map (major type 5) - let first_byte = op_cbor[0]; - let major_type = (first_byte >> 5) & 0x07; - if major_type != 5 { - return Err(atoms::parse_failed()); - } - - // M10 PoC: Return dummy block ID as binary (u64 = 1) - let block_id: u64 = 1; - let result = block_id.to_be_bytes().to_vec(); - - Ok(result) -} - -/// Get database schema -#[rustler::nif] -fn schema(_db: ResourceArc) -> Vec { - // M10 PoC: Return empty CBOR map - vec![0xa0] // CBOR: {} -} - -/// Get journal entries since a sequence number -#[rustler::nif] -fn journal( - _db: ResourceArc, - _since: i64, -) -> Vec { - // M10 PoC: Return empty CBOR array - vec![0x80] // CBOR: [] -} diff --git a/lithoglyph/glyphbase/server/src/abi/Foreign.idr b/lithoglyph/glyphbase/server/src/abi/Foreign.idr deleted file mode 100644 index 9c6213fa..00000000 --- a/lithoglyph/glyphbase/server/src/abi/Foreign.idr +++ /dev/null @@ -1,219 +0,0 @@ --- SPDX-License-Identifier: MPL-2.0 --- Lithoglyph ABI Foreign Function Interface Declarations --- --- This module declares the C-compatible FFI functions that will be --- implemented in Zig. All functions use the C calling convention and --- follow the ABI layout guarantees from Layout.idr. - -module Foreign - -import Types -import Layout -import Data.Bits -import Data.Buffer - -%default total - --- Foreign function declarations --- These will be implemented in ffi/zig/src/main.zig - --- Get NIF version (major, minor, patch) -%foreign "C:lithoglyph_nif_version, liblithoglyph_nif" -prim__nif_version : PrimIO (Bits8, Bits8, Bits8) - -export -nifVersion : IO Version -nifVersion = do - (maj, min, pat) <- primIO prim__nif_version - pure $ MkVersion maj min pat - --- Open database connection --- Returns NULL on error (checked by constructor proof) -%foreign "C:lithoglyph_nif_db_open, liblithoglyph_nif" -prim__db_open : String -> PrimIO Bits64 - -export -dbOpen : (path : DbPath) -> IO (FFIResult DbHandle) -dbOpen (MkDbPath pathStr) = do - ptr <- primIO (prim__db_open pathStr) - if ptr == 0 - then pure $ Error "Failed to open database" - else case decSo (ptr /= 0) of - Yes prf => pure $ Ok (MkDbHandle ptr @{prf}) - No _ => pure $ Error "Null pointer returned" - --- Close database connection -%foreign "C:lithoglyph_nif_db_close, liblithoglyph_nif" -prim__db_close : Bits64 -> PrimIO Int - -export -dbClose : DbHandle -> IO (FFIResult ()) -dbClose handle = do - result <- primIO (prim__db_close (dbHandlePtr handle)) - if result == 0 - then pure $ Ok () - else pure $ Error "Failed to close database" - --- Begin transaction --- mode: 0 = ReadOnly, 1 = ReadWrite -%foreign "C:lithoglyph_nif_txn_begin, liblithoglyph_nif" -prim__txn_begin : Bits64 -> Bits32 -> PrimIO Bits64 - -export -txnBegin : DbHandle -> TxnMode -> IO (FFIResult TxnHandle) -txnBegin db mode = do - let modeInt = txnModeToInt mode - ptr <- primIO (prim__txn_begin (dbHandlePtr db) modeInt) - if ptr == 0 - then pure $ Error "Failed to begin transaction" - else case decSo (ptr /= 0) of - Yes prf => pure $ Ok (MkTxnHandle ptr @{prf}) - No _ => pure $ Error "Null pointer returned" - --- Commit transaction -%foreign "C:lithoglyph_nif_txn_commit, liblithoglyph_nif" -prim__txn_commit : Bits64 -> PrimIO Int - -export -txnCommit : TxnHandle -> IO (FFIResult ()) -txnCommit txn = do - result <- primIO (prim__txn_commit (txnHandlePtr txn)) - if result == 0 - then pure $ Ok () - else pure $ Error "Failed to commit transaction" - --- Abort transaction -%foreign "C:lithoglyph_nif_txn_abort, liblithoglyph_nif" -prim__txn_abort : Bits64 -> PrimIO Int - -export -txnAbort : TxnHandle -> IO (FFIResult ()) -txnAbort txn = do - result <- primIO (prim__txn_abort (txnHandlePtr txn)) - if result == 0 - then pure $ Ok () - else pure $ Error "Failed to abort transaction" - --- Apply operation to transaction --- Returns block ID and optional provenance hash --- Input: transaction handle, operation CBOR bytes, operation length --- Output: block_id (u64), has_provenance (bool), provenance_hash (32 bytes if present) -%foreign "C:lithoglyph_nif_apply, liblithoglyph_nif" -prim__apply : Bits64 -> Buffer -> Bits32 -> Bits64 -> Bits32 -> Buffer -> PrimIO Int - -export -applyOperation : TxnHandle -> OperationData -> IO (FFIResult (BlockId, Maybe (List Bits8))) -applyOperation txn (MkOperationData bytes len) = do - -- Allocate buffers for input and output - opBuf <- newBuffer (cast len) - -- Copy operation bytes to buffer - let _ = writeBufferBytes opBuf 0 bytes len - - -- Allocate output buffer for provenance hash (32 bytes) - provBuf <- newBuffer 32 - blockIdRef <- newBuffer 8 -- u64 output - hasProvRef <- newBuffer 4 -- bool output (u32) - - result <- primIO (prim__apply - (txnHandlePtr txn) - opBuf - (cast len) - blockIdRef - hasProvRef - provBuf) - - if result /= 0 - then pure $ Error "Failed to apply operation" - else do - -- Read block ID - blockId <- readBufferBits64 blockIdRef 0 - -- Check if provenance present - hasProv <- readBufferBits32 hasProvRef 0 - - if hasProv == 0 - then pure $ Ok (MkBlockId blockId, Nothing) - else do - -- Read provenance hash (32 bytes) - provBytes <- readBufferBytes provBuf 0 32 - pure $ Ok (MkBlockId blockId, Just provBytes) - --- Get database schema (CBOR-encoded) --- Returns: length (u32), data buffer -%foreign "C:lithoglyph_nif_schema, liblithoglyph_nif" -prim__schema : Bits64 -> Buffer -> Bits32 -> PrimIO Bits32 - -export -getSchema : DbHandle -> IO (FFIResult SchemaData) -getSchema db = do - -- Allocate output buffer (max 1MB for schema) - schemaBuf <- newBuffer (1024 * 1024) - - len <- primIO (prim__schema (dbHandlePtr db) schemaBuf (1024 * 1024)) - - if len == 0 - then pure $ Error "Failed to get schema" - else do - schemaBytes <- readBufferBytes schemaBuf 0 (cast len) - case decEq (length schemaBytes) (cast len) of - Yes prf => pure $ Ok (MkSchemaData schemaBytes (cast len) @{prf}) - No _ => pure $ Error "Schema length mismatch" - --- Get journal entries since timestamp --- Returns: length (u32), data buffer (CBOR-encoded array) -%foreign "C:lithoglyph_nif_journal, liblithoglyph_nif" -prim__journal : Bits64 -> Bits64 -> Buffer -> Bits32 -> PrimIO Bits32 - -export -getJournal : DbHandle -> Timestamp -> IO (FFIResult JournalData) -getJournal db (MkTimestamp since) = do - -- Allocate output buffer (max 10MB for journal) - journalBuf <- newBuffer (10 * 1024 * 1024) - - len <- primIO (prim__journal (dbHandlePtr db) since journalBuf (10 * 1024 * 1024)) - - if len == 0 - then pure $ Error "Failed to get journal" - else do - journalBytes <- readBufferBytes journalBuf 0 (cast len) - case decEq (length journalBytes) (cast len) of - Yes prf => pure $ Ok (MkJournalData journalBytes (cast len) @{prf}) - No _ => pure $ Error "Journal length mismatch" - --- Helper: read bytes from buffer -readBufferBytes : Buffer -> (offset : Int) -> (len : Nat) -> IO (List Bits8) -readBufferBytes buf offset Z = pure [] -readBufferBytes buf offset (S k) = do - byte <- getBits8 buf offset - rest <- readBufferBytes buf (offset + 1) k - pure (byte :: rest) - --- Helper: write bytes to buffer -writeBufferBytes : Buffer -> (offset : Int) -> (bytes : List Bits8) -> (len : Nat) -> IO () -writeBufferBytes buf offset [] Z = pure () -writeBufferBytes buf offset (b :: bs) (S k) = do - setBits8 buf offset b - writeBufferBytes buf (offset + 1) bs k -writeBufferBytes _ _ _ _ = pure () -- length mismatch, ignore - --- Helper: read u64 from buffer (little-endian) -readBufferBits64 : Buffer -> (offset : Int) -> IO Bits64 -readBufferBits64 buf offset = do - b0 <- getBits8 buf (offset + 0) - b1 <- getBits8 buf (offset + 1) - b2 <- getBits8 buf (offset + 2) - b3 <- getBits8 buf (offset + 3) - b4 <- getBits8 buf (offset + 4) - b5 <- getBits8 buf (offset + 5) - b6 <- getBits8 buf (offset + 6) - b7 <- getBits8 buf (offset + 7) - pure $ (cast b0) + (cast b1 `shiftL` 8) + (cast b2 `shiftL` 16) + (cast b3 `shiftL` 24) + - (cast b4 `shiftL` 32) + (cast b5 `shiftL` 40) + (cast b6 `shiftL` 48) + (cast b7 `shiftL` 56) - --- Helper: read u32 from buffer (little-endian) -readBufferBits32 : Buffer -> (offset : Int) -> IO Bits32 -readBufferBits32 buf offset = do - b0 <- getBits8 buf (offset + 0) - b1 <- getBits8 buf (offset + 1) - b2 <- getBits8 buf (offset + 2) - b3 <- getBits8 buf (offset + 3) - pure $ (cast b0) + (cast b1 `shiftL` 8) + (cast b2 `shiftL` 16) + (cast b3 `shiftL` 24) diff --git a/lithoglyph/glyphbase/server/src/abi/Layout.idr b/lithoglyph/glyphbase/server/src/abi/Layout.idr deleted file mode 100644 index f76a6e8e..00000000 --- a/lithoglyph/glyphbase/server/src/abi/Layout.idr +++ /dev/null @@ -1,181 +0,0 @@ --- SPDX-License-Identifier: MPL-2.0 --- Lith ABI Memory Layout Verification --- --- This module provides compile-time proofs of memory layout correctness, --- ensuring that struct sizes, alignments, and padding are correct across --- all target platforms (Linux x86_64, Linux ARM64, macOS x86_64, macOS ARM64). - -module Layout - -import Types -import Data.Bits -import Data.Nat - -%default total - --- Platform-specific alignment requirements -public export -data Platform : Type where - Linux_x86_64 : Platform - Linux_ARM64 : Platform - MacOS_x86_64 : Platform - MacOS_ARM64 : Platform - Windows_x86_64 : Platform - --- Pointer size for platform (in bytes) -public export -ptrSize : Platform -> Nat -ptrSize Linux_x86_64 = 8 -ptrSize Linux_ARM64 = 8 -ptrSize MacOS_x86_64 = 8 -ptrSize MacOS_ARM64 = 8 -ptrSize Windows_x86_64 = 8 - --- Natural alignment for pointers (must be power of 2) -public export -ptrAlignment : Platform -> Nat -ptrAlignment Linux_x86_64 = 8 -ptrAlignment Linux_ARM64 = 8 -ptrAlignment MacOS_x86_64 = 8 -ptrAlignment MacOS_ARM64 = 8 -ptrAlignment Windows_x86_64 = 8 - --- Proof that pointer size equals pointer alignment on all platforms -public export -0 ptrSizeEqualsAlignment : (p : Platform) -> ptrSize p = ptrAlignment p -ptrSizeEqualsAlignment Linux_x86_64 = Refl -ptrSizeEqualsAlignment Linux_ARM64 = Refl -ptrSizeEqualsAlignment MacOS_x86_64 = Refl -ptrSizeEqualsAlignment MacOS_ARM64 = Refl -ptrSizeEqualsAlignment Windows_x86_64 = Refl - --- Size of opaque handle types (just a pointer) -public export -dbHandleSize : Platform -> Nat -dbHandleSize p = ptrSize p - -public export -txnHandleSize : Platform -> Nat -txnHandleSize p = ptrSize p - --- Alignment of opaque handle types -public export -dbHandleAlignment : Platform -> Nat -dbHandleAlignment p = ptrAlignment p - -public export -txnHandleAlignment : Platform -> Nat -txnHandleAlignment p = ptrAlignment p - --- Version struct layout --- { major: u8, minor: u8, patch: u8 } --- Total size: 3 bytes (no padding in C with explicit packing) -public export -versionSize : Nat -versionSize = 3 - -public export -versionAlignment : Nat -versionAlignment = 1 -- byte-aligned - --- Proof that Version struct has correct size -public export -0 versionSizeCorrect : versionSize = 3 -versionSizeCorrect = Refl - --- Block ID size (u64) -public export -blockIdSize : Nat -blockIdSize = 8 - -public export -blockIdAlignment : Nat -blockIdAlignment = 8 - --- Timestamp size (u64) -public export -timestampSize : Nat -timestampSize = 8 - -public export -timestampAlignment : Nat -timestampAlignment = 8 - --- Proof that u64 types have correct size and alignment -public export -0 blockIdSizeCorrect : blockIdSize = 8 -blockIdSizeCorrect = Refl - -public export -0 timestampSizeCorrect : timestampSize = 8 -timestampSizeCorrect = Refl - --- FFI result enum tag (0 = Ok, 1 = Error) --- Represented as u32 for ABI stability -public export -resultTagSize : Nat -resultTagSize = 4 - -public export -resultTagAlignment : Nat -resultTagAlignment = 4 - --- Helper: round up to next multiple of alignment -public export -roundUp : Nat -> Nat -> Nat -roundUp size align = - let remainder = size `mod` align - in if remainder == 0 - then size - else size + (align - remainder) - --- Proof that roundUp preserves divisibility -public export -0 roundUpDivisible : (size : Nat) -> (align : Nat) -> - {auto 0 alignNonZero : So (align > 0)} -> - (roundUp size align `mod` align = 0) --- PROOF_TODO: Replace cast with actual proof -roundUpDivisible size align = cast () -- TODO: formal proof - --- Calculate offset of next field after current field -public export -nextFieldOffset : (currentSize : Nat) -> (nextAlignment : Nat) -> Nat -nextFieldOffset currentSize nextAlignment = roundUp currentSize nextAlignment - --- Proof: opaque handles maintain pointer alignment across platforms -public export -0 handleAlignmentCorrect : (p : Platform) -> - dbHandleAlignment p = ptrAlignment p -handleAlignmentCorrect _ = Refl - --- ABI stability guarantee: opaque handles are always pointer-sized -public export -0 handleSizeStable : (p1 : Platform) -> (p2 : Platform) -> - dbHandleSize p1 = dbHandleSize p2 -handleSizeStable Linux_x86_64 Linux_x86_64 = Refl -handleSizeStable Linux_x86_64 Linux_ARM64 = Refl -handleSizeStable Linux_x86_64 MacOS_x86_64 = Refl -handleSizeStable Linux_x86_64 MacOS_ARM64 = Refl -handleSizeStable Linux_x86_64 Windows_x86_64 = Refl -handleSizeStable Linux_ARM64 Linux_x86_64 = Refl -handleSizeStable Linux_ARM64 Linux_ARM64 = Refl -handleSizeStable Linux_ARM64 MacOS_x86_64 = Refl -handleSizeStable Linux_ARM64 MacOS_ARM64 = Refl -handleSizeStable Linux_ARM64 Windows_x86_64 = Refl -handleSizeStable MacOS_x86_64 Linux_x86_64 = Refl -handleSizeStable MacOS_x86_64 Linux_ARM64 = Refl -handleSizeStable MacOS_x86_64 MacOS_x86_64 = Refl -handleSizeStable MacOS_x86_64 MacOS_ARM64 = Refl -handleSizeStable MacOS_x86_64 Windows_x86_64 = Refl -handleSizeStable MacOS_ARM64 Linux_x86_64 = Refl -handleSizeStable MacOS_ARM64 Linux_ARM64 = Refl -handleSizeStable MacOS_ARM64 MacOS_x86_64 = Refl -handleSizeStable MacOS_ARM64 MacOS_ARM64 = Refl -handleSizeStable MacOS_ARM64 Windows_x86_64 = Refl -handleSizeStable Windows_x86_64 Linux_x86_64 = Refl -handleSizeStable Windows_x86_64 Linux_ARM64 = Refl -handleSizeStable Windows_x86_64 MacOS_x86_64 = Refl -handleSizeStable Windows_x86_64 MacOS_ARM64 = Refl -handleSizeStable Windows_x86_64 Windows_x86_64 = Refl - --- Total: All ABI types maintain stable layout across all supported platforms diff --git a/lithoglyph/glyphbase/server/src/abi/Types.idr b/lithoglyph/glyphbase/server/src/abi/Types.idr deleted file mode 100644 index 0e9e92f8..00000000 --- a/lithoglyph/glyphbase/server/src/abi/Types.idr +++ /dev/null @@ -1,124 +0,0 @@ --- SPDX-License-Identifier: MPL-2.0 --- Lith ABI Type Definitions with Dependent Type Proofs --- --- This module defines the ABI types for the Lith/Lithoglyph database --- with formal verification of memory safety and layout correctness. - -module Types - -import Data.Bits -import Data.So - -%default total - --- Non-null pointer guarantee at type level --- The So proof ensures the pointer value cannot be zero -public export -data DbHandle : Type where - MkDbHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> DbHandle - -public export -data TxnHandle : Type where - MkTxnHandle : (ptr : Bits64) -> {auto 0 nonNull : So (ptr /= 0)} -> TxnHandle - --- Transaction mode enumeration --- Corresponds to Gleam TransactionMode type -public export -data TxnMode : Type where - ReadOnly : TxnMode - ReadWrite : TxnMode - --- Convert TxnMode to integer for FFI -public export -txnModeToInt : TxnMode -> Bits32 -txnModeToInt ReadOnly = 0 -txnModeToInt ReadWrite = 1 - --- Result type for operations that can fail --- Matches Erlang {ok, Value} | {error, Reason} convention -public export -data FFIResult : Type -> Type where - Ok : (value : a) -> FFIResult a - Error : (reason : String) -> FFIResult a - --- Version tuple (major, minor, patch) -public export -record Version where - constructor MkVersion - major : Bits8 - minor : Bits8 - patch : Bits8 - --- Proof that version numbers are valid (0-255) -public export -0 validVersion : Version -> Type -validVersion v = (v.major <= 255, v.minor <= 255, v.patch <= 255) - --- Block ID returned from apply operation --- Stone-carved database block identifier -public export -data BlockId : Type where - MkBlockId : (id : Bits64) -> BlockId - --- Timestamp for journal queries (Unix epoch microseconds) -public export -data Timestamp : Type where - MkTimestamp : (micros : Bits64) -> Timestamp - --- CBOR-encoded operation data --- Raw bytes representing serialized Lith operation -public export -data OperationData : Type where - MkOperationData : (bytes : List Bits8) -> (len : Nat) -> - {auto 0 lengthCorrect : length bytes = len} -> - OperationData - --- Schema data (CBOR-encoded) -public export -data SchemaData : Type where - MkSchemaData : (bytes : List Bits8) -> (len : Nat) -> - {auto 0 lengthCorrect : length bytes = len} -> - SchemaData - --- Journal data (CBOR-encoded) -public export -data JournalData : Type where - MkJournalData : (bytes : List Bits8) -> (len : Nat) -> - {auto 0 lengthCorrect : length bytes = len} -> - JournalData - --- Database path (null-terminated C string) -public export -record DbPath where - constructor MkDbPath - path : String - {auto 0 nonEmpty : So (length path > 0)} - --- Extract raw pointer value (for FFI) --- This is safe because the constructor proves non-null -public export -dbHandlePtr : DbHandle -> Bits64 -dbHandlePtr (MkDbHandle ptr) = ptr - -public export -txnHandlePtr : TxnHandle -> Bits64 -txnHandlePtr (MkTxnHandle ptr) = ptr - --- Functor instance for FFIResult -public export -Functor FFIResult where - map f (Ok x) = Ok (f x) - map _ (Error e) = Error e - --- Monad instance for FFIResult -public export -Monad FFIResult where - (Ok x) >>= f = f x - (Error e) >>= _ = Error e - -public export -Applicative FFIResult where - pure = Ok - (Ok f) <*> (Ok x) = Ok (f x) - (Error e) <*> _ = Error e - _ <*> (Error e) = Error e diff --git a/lithoglyph/glyphbase/server/src/glyphbase_server.gleam b/lithoglyph/glyphbase/server/src/glyphbase_server.gleam deleted file mode 100644 index eca162f0..00000000 --- a/lithoglyph/glyphbase/server/src/glyphbase_server.gleam +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Glyphbase API Server - -import envoy -import lithoglyph/client -import gleam/erlang/process -import gleam/http -import gleam/int -import gleam/io -import mist -import router.{type Context, Context} -import wisp -import wisp/wisp_mist - -/// Default database path -const default_db_path = "./data/glyphbase.db" - -pub fn main() { - io.println("Glyphbase Server v0.1.0") - - // Get database path from environment or use default - let db_path = get_env_or_default("GLYPHBASE_DB_PATH", default_db_path) - io.println("Database path: " <> db_path) - - // Open database connection - io.println("Connecting to Lith...") - let db = case client.connect(db_path) { - Ok(conn) -> { - io.println("Connected to Lith successfully") - conn - } - Error(e) -> { - io.println("Warning: Failed to connect to Lith: " <> format_error(e)) - io.println("Server will start but database operations will fail") - io.println("Make sure Lith NIF is loaded and database exists") - // Return a placeholder - will fail on actual DB operations - panic as "Database connection required" - } - } - - // Create context for handlers - let ctx = Context(db: db) - - // Configure secret key for wisp - let secret_key_base = wisp.random_string(64) - - // Get port from environment or use default - let port = get_port_or_default(8080) - io.println("Starting on http://localhost:" <> int_to_string(port)) - - // Start the HTTP server - let assert Ok(_) = - wisp_mist.handler(handle_request(_, secret_key_base, ctx), secret_key_base) - |> mist.new - |> mist.port(port) - |> mist.start - - io.println("Server running!") - process.sleep_forever() -} - -fn handle_request( - req: wisp.Request, - _secret: String, - ctx: Context, -) -> wisp.Response { - // Handle preflight requests for CORS - case req.method { - http.Options -> { - wisp.ok() - |> wisp.set_header("Access-Control-Allow-Origin", "*") - |> wisp.set_header( - "Access-Control-Allow-Methods", - "GET, POST, PATCH, DELETE, OPTIONS", - ) - |> wisp.set_header("Access-Control-Allow-Headers", "Content-Type, Authorization") - } - _ -> { - router.handle_request(req, ctx) - |> wisp.set_header("Access-Control-Allow-Origin", "*") - } - } -} - -fn format_error(error: client.LithError) -> String { - case error { - client.ConnectionError(msg) -> "Connection error: " <> msg - client.TransactionError(msg) -> "Transaction error: " <> msg - client.QueryError(msg) -> "Query error: " <> msg - client.ValidationError(msg) -> "Validation error: " <> msg - client.ProvenanceError(msg) -> "Provenance error: " <> msg - client.NotFound(entity, id) -> entity <> " not found: " <> id - client.PermissionDenied(action) -> "Permission denied: " <> action - client.NifNotLoaded -> "NIF not loaded" - client.NifError(reason) -> "NIF error: " <> reason - client.ParseFailed -> "CBOR parse failed" - client.InvalidHandle -> "Invalid handle" - client.PathTraversal(path) -> "Path traversal rejected: " <> path - } -} - -// ============================================================ -// Environment helpers -// ============================================================ - -fn get_env_or_default(name: String, default: String) -> String { - case envoy.get(name) { - Ok(value) -> - case value { - "" -> default - _ -> value - } - Error(_) -> default - } -} - -fn get_port_or_default(default: Int) -> Int { - case envoy.get("PORT") { - Ok(value) -> { - case int.parse(value) { - Ok(port) -> port - Error(_) -> default - } - } - Error(_) -> default - } -} - -fn int_to_string(n: Int) -> String { - int.to_string(n) -} diff --git a/lithoglyph/glyphbase/server/src/lith_nif.erl b/lithoglyph/glyphbase/server/src/lith_nif.erl deleted file mode 100644 index 03d00e2a..00000000 --- a/lithoglyph/glyphbase/server/src/lith_nif.erl +++ /dev/null @@ -1,57 +0,0 @@ -% SPDX-License-Identifier: MPL-2.0 -% SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -% -% lith_nif.erl - Erlang NIF loader module for Lithoglyph -% -% This module loads the Zig-compiled shared library and provides stub -% functions that are replaced by the NIF at load time. The module name -% must match the .name field in the Zig ErlNifEntry ("lith_nif"). - --module(lith_nif). --export([version/0, db_open/1, db_close/1, txn_begin/2, txn_commit/1, - txn_abort/1, apply/2, schema/1, journal/2]). --on_load(load_nif/0). - -load_nif() -> - PrivDir = case code:priv_dir(glyphbase_server) of - {error, _} -> - %% Development fallback — check native/priv and ./priv - case filelib:is_dir("native/priv") of - true -> "native/priv"; - false -> "./priv" - end; - Dir -> - Dir - end, - NifPath = filename:join(PrivDir, "liblithoglyph_nif"), - erlang:load_nif(NifPath, 0). - -%% Stub functions replaced by NIF at load time. -%% If these are called, the NIF failed to load. - -version() -> - erlang:nif_error(nif_not_loaded). - -db_open(_Path) -> - erlang:nif_error(nif_not_loaded). - -db_close(_Db) -> - erlang:nif_error(nif_not_loaded). - -txn_begin(_Db, _Mode) -> - erlang:nif_error(nif_not_loaded). - -txn_commit(_Txn) -> - erlang:nif_error(nif_not_loaded). - -txn_abort(_Txn) -> - erlang:nif_error(nif_not_loaded). - -apply(_Txn, _OpCbor) -> - erlang:nif_error(nif_not_loaded). - -schema(_Db) -> - erlang:nif_error(nif_not_loaded). - -journal(_Db, _Since) -> - erlang:nif_error(nif_not_loaded). diff --git a/lithoglyph/glyphbase/server/src/lithoglyph.gleam b/lithoglyph/glyphbase/server/src/lithoglyph.gleam deleted file mode 100644 index a3dc04a4..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph.gleam +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// lithoglyph.gleam - Public API for Lithoglyph database -// -// This module re-exports the client API for convenient access. -// The real NIF FFI layer is in lithoglyph/nif_ffi.gleam, -// and the typed client wrapper is in lithoglyph/client.gleam. - -import gleam/option.{type Option} -import lithoglyph/client.{ - type Connection, type LithError, type LithResult, type Transaction, - type TransactionMode, ReadOnly, ReadWrite, -} - -/// Re-export core types -pub type Db = - Connection - -pub type Txn = - Transaction - -pub type Mode = - TransactionMode - -/// Re-export error type -pub type Error = - LithError - -/// Get Lithoglyph version tuple -pub fn version() -> #(Int, Int, Int) { - client.version() -} - -/// Open a connection to a Lithoglyph database -pub fn open(path: String) -> LithResult(Connection) { - client.connect(path) -} - -/// Close a connection -pub fn close(conn: Connection) -> LithResult(Nil) { - client.disconnect(conn) -} - -/// Begin a read-only transaction -pub fn begin_read(conn: Connection) -> LithResult(Transaction) { - client.begin_transaction(conn, ReadOnly) -} - -/// Begin a read-write transaction -pub fn begin_write(conn: Connection) -> LithResult(Transaction) { - client.begin_transaction(conn, ReadWrite) -} - -/// Commit a transaction -pub fn commit(txn: Transaction) -> LithResult(Nil) { - client.commit(txn) -} - -/// Abort a transaction -pub fn abort(txn: Transaction) -> LithResult(Nil) { - client.abort(txn) -} - -/// Apply a CBOR-encoded operation within a transaction -pub fn apply( - txn: Transaction, - operation: BitArray, -) -> LithResult(#(BitArray, Option(BitArray))) { - client.apply_operation(txn, operation) -} - -/// Get database schema (CBOR-encoded) -pub fn schema(conn: Connection) -> LithResult(BitArray) { - client.get_schema(conn) -} - -/// Get journal entries since a sequence number (CBOR-encoded) -pub fn journal(conn: Connection, since: Int) -> LithResult(BitArray) { - client.get_journal(conn, since) -} - -/// Execute an operation in a transaction with automatic commit/abort -pub fn with_transaction( - conn: Connection, - mode: TransactionMode, - operation: fn(Transaction) -> LithResult(a), -) -> LithResult(a) { - client.with_transaction(conn, mode, operation) -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/cbor.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/cbor.gleam deleted file mode 100644 index c5be1c0d..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/cbor.gleam +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Simple CBOR encoder for Lithoglyph operations - -import gleam/bit_array -import gleam/list -import gleam/option.{type Option, None, Some} - -/// CBOR major types -const major_unsigned = 0 - -const major_negative = 1 - -const major_bytes = 2 - -const major_text = 3 - -const major_array = 4 - -const major_map = 5 - -const major_simple = 7 - -/// Encode an unsigned integer -pub fn encode_unsigned(n: Int) -> BitArray { - case n { - _ if n < 24 -> <<{ major_unsigned * 32 + n }>> - _ if n < 256 -> <<{ major_unsigned * 32 + 24 }, n:8>> - _ if n < 65_536 -> <<{ major_unsigned * 32 + 25 }, n:16>> - _ if n < 4_294_967_296 -> <<{ major_unsigned * 32 + 26 }, n:32>> - _ -> <<{ major_unsigned * 32 + 27 }, n:64>> - } -} - -/// Encode a negative integer -pub fn encode_negative(n: Int) -> BitArray { - let m = -1 - n - case m { - _ if m < 24 -> <<{ major_negative * 32 + m }>> - _ if m < 256 -> <<{ major_negative * 32 + 24 }, m:8>> - _ if m < 65_536 -> <<{ major_negative * 32 + 25 }, m:16>> - _ if m < 4_294_967_296 -> <<{ major_negative * 32 + 26 }, m:32>> - _ -> <<{ major_negative * 32 + 27 }, m:64>> - } -} - -/// Encode an integer (positive or negative) -pub fn encode_int(n: Int) -> BitArray { - case n >= 0 { - True -> encode_unsigned(n) - False -> encode_negative(n) - } -} - -/// Encode a float (64-bit double precision) -pub fn encode_float(f: Float) -> BitArray { - <<{ major_simple * 32 + 27 }, f:64-float>> -} - -/// Encode a byte string -pub fn encode_bytes(data: BitArray) -> BitArray { - let len = bit_array.byte_size(data) - let header = encode_length(major_bytes, len) - bit_array.concat([header, data]) -} - -/// Encode a UTF-8 text string -pub fn encode_text(text: String) -> BitArray { - let bytes = <> - let len = bit_array.byte_size(bytes) - let header = encode_length(major_text, len) - bit_array.concat([header, bytes]) -} - -/// Encode an array header (call this, then encode each element) -pub fn encode_array_header(len: Int) -> BitArray { - encode_length(major_array, len) -} - -/// Encode an array with pre-encoded elements -pub fn encode_array(elements: List(BitArray)) -> BitArray { - let header = encode_array_header(list.length(elements)) - bit_array.concat([header, ..elements]) -} - -/// Encode a map header (call this, then encode key-value pairs) -pub fn encode_map_header(len: Int) -> BitArray { - encode_length(major_map, len) -} - -/// Encode a map with pre-encoded key-value pairs -pub fn encode_map(pairs: List(#(BitArray, BitArray))) -> BitArray { - let header = encode_map_header(list.length(pairs)) - let pairs_encoded = - pairs - |> list.flat_map(fn(pair) { [pair.0, pair.1] }) - bit_array.concat([header, ..pairs_encoded]) -} - -/// Encode a boolean -pub fn encode_bool(b: Bool) -> BitArray { - case b { - False -> <<{ major_simple * 32 + 20 }>> - True -> <<{ major_simple * 32 + 21 }>> - } -} - -/// Encode null -pub fn encode_null() -> BitArray { - <<{ major_simple * 32 + 22 }>> -} - -/// Encode an optional value -pub fn encode_optional(opt: Option(a), encoder: fn(a) -> BitArray) -> BitArray { - case opt { - None -> encode_null() - Some(value) -> encoder(value) - } -} - -/// Helper to encode length for a major type -fn encode_length(major: Int, len: Int) -> BitArray { - case len { - _ if len < 24 -> <<{ major * 32 + len }>> - _ if len < 256 -> <<{ major * 32 + 24 }, len:8>> - _ if len < 65_536 -> <<{ major * 32 + 25 }, len:16>> - _ if len < 4_294_967_296 -> <<{ major * 32 + 26 }, len:32>> - _ -> <<{ major * 32 + 27 }, len:64>> - } -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/client.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/client.gleam deleted file mode 100644 index aaac4310..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/client.gleam +++ /dev/null @@ -1,304 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// Lith client - Safe Gleam interface to Lith via NIF -// -// All NIF results are properly decoded from Erlang tuples. -// No unsafe_coerce — resource handles are extracted via erlang:element/2 -// which is safe because the NIF guarantees the tuple structure. - -import gleam/bit_array -import gleam/dynamic -import gleam/option.{type Option, None} -import lithoglyph/nif_ffi -import lithoglyph/safety - -/// Lith database handle (opaque reference from NIF) -pub opaque type Connection { - Connection(handle: nif_ffi.DbHandle) -} - -/// Lith transaction handle (opaque reference from NIF) -pub opaque type Transaction { - Transaction(handle: nif_ffi.TxnHandle, conn: Connection) -} - -/// Transaction mode -pub type TransactionMode { - ReadOnly - ReadWrite -} - -/// Lith error types -pub type LithError { - ConnectionError(message: String) - TransactionError(message: String) - QueryError(message: String) - ValidationError(message: String) - ProvenanceError(message: String) - NotFound(entity: String, id: String) - PermissionDenied(action: String) - NifNotLoaded - NifError(reason: String) - ParseFailed - InvalidHandle - PathTraversal(path: String) -} - -/// Result type for Lith operations -pub type LithResult(a) = - Result(a, LithError) - -// ============================================================ -// Path Validation -// ============================================================ - -/// Validate a database path for directory traversal attacks. -/// Uses proven-compatible SafePath checks (null bytes + ".." detection). -fn validate_path(path: String) -> LithResult(String) { - case safety.path_has_traversal(path) { - Ok(True) -> Error(PathTraversal(path: path)) - Ok(False) -> Ok(path) - Error(reason) -> Error(ValidationError(message: reason)) - } -} - -// ============================================================ -// Public API -// ============================================================ - -/// Get Lith version -pub fn version() -> #(Int, Int, Int) { - nif_ffi.nif_version() -} - -/// Open a connection to a Lith database. -/// The path is validated against directory traversal before opening. -pub fn connect(path: String) -> LithResult(Connection) { - case validate_path(path) { - Error(e) -> Error(e) - Ok(safe_path) -> { - let path_binary = bit_array.from_string(safe_path) - let result = nif_ffi.nif_db_open(path_binary) - case decode_ok_result(result) { - Ok(handle_dyn) -> { - let handle = coerce_to_db_handle(handle_dyn) - Ok(Connection(handle: handle)) - } - Error(reason) -> Error(ConnectionError(message: reason)) - } - } - } -} - -/// Close a Lith connection -pub fn disconnect(conn: Connection) -> LithResult(Nil) { - let result = nif_ffi.nif_db_close(conn.handle) - case decode_atom_or_error(result) { - Ok(_) -> Ok(Nil) - Error(reason) -> Error(ConnectionError(message: reason)) - } -} - -/// Begin a transaction -pub fn begin_transaction( - conn: Connection, - mode: TransactionMode, -) -> LithResult(Transaction) { - let mode_binary = case mode { - ReadOnly -> <<"read_only":utf8>> - ReadWrite -> <<"read_write":utf8>> - } - - let result = nif_ffi.nif_txn_begin(conn.handle, mode_binary) - case decode_ok_result(result) { - Ok(handle_dyn) -> { - let handle = coerce_to_txn_handle(handle_dyn) - Ok(Transaction(handle: handle, conn: conn)) - } - Error(reason) -> Error(TransactionError(message: reason)) - } -} - -/// Commit a transaction -pub fn commit(txn: Transaction) -> LithResult(Nil) { - let result = nif_ffi.nif_txn_commit(txn.handle) - case decode_atom_or_error(result) { - Ok(_) -> Ok(Nil) - Error(reason) -> Error(TransactionError(message: reason)) - } -} - -/// Abort a transaction -pub fn abort(txn: Transaction) -> LithResult(Nil) { - let result = nif_ffi.nif_txn_abort(txn.handle) - case decode_atom_or_error(result) { - Ok(_) -> Ok(Nil) - Error(reason) -> Error(TransactionError(message: reason)) - } -} - -/// Apply an operation within a transaction. -/// The operation should be CBOR-encoded. -/// Returns (BlockId, Optional Provenance) -pub fn apply_operation( - txn: Transaction, - operation: BitArray, -) -> LithResult(#(BitArray, Option(BitArray))) { - let result = nif_ffi.nif_apply(txn.handle, operation) - case decode_ok_result(result) { - Ok(block_id_dyn) -> { - let block_id = coerce_to_bit_array(block_id_dyn) - Ok(#(block_id, None)) - } - Error(reason) -> Error(QueryError(message: reason)) - } -} - -/// Get database schema (CBOR-encoded) -pub fn get_schema(conn: Connection) -> LithResult(BitArray) { - let result = nif_ffi.nif_schema(conn.handle) - case decode_ok_result(result) { - Ok(data_dyn) -> Ok(coerce_to_bit_array(data_dyn)) - Error(reason) -> Error(QueryError(message: reason)) - } -} - -/// Get journal entries since a sequence number (CBOR-encoded) -pub fn get_journal(conn: Connection, since: Int) -> LithResult(BitArray) { - let result = nif_ffi.nif_journal(conn.handle, since) - case decode_ok_result(result) { - Ok(data_dyn) -> Ok(coerce_to_bit_array(data_dyn)) - Error(reason) -> Error(QueryError(message: reason)) - } -} - -// ============================================================ -// High-Level Operations -// ============================================================ - -/// Execute an operation in a transaction with automatic commit/abort -pub fn with_transaction( - conn: Connection, - mode: TransactionMode, - operation: fn(Transaction) -> LithResult(a), -) -> LithResult(a) { - case begin_transaction(conn, mode) { - Ok(txn) -> { - case operation(txn) { - Ok(result) -> { - case commit(txn) { - Ok(_) -> Ok(result) - Error(e) -> { - let _ = abort(txn) - Error(e) - } - } - } - Error(e) -> { - let _ = abort(txn) - Error(e) - } - } - } - Error(e) -> Error(e) - } -} - -// ============================================================ -// Erlang Term Decoding (replaces unsafe_coerce) -// ============================================================ - -/// Extract element from an Erlang tuple by 1-based index. -/// This is safe because we only call it after verifying tuple structure. -@external(erlang, "erlang", "element") -fn erlang_element(index: Int, tuple: dynamic.Dynamic) -> dynamic.Dynamic - -/// Get the size of an Erlang tuple -@external(erlang, "erlang", "tuple_size") -fn erlang_tuple_size(tuple: dynamic.Dynamic) -> Int - -/// Check if a dynamic value is a specific atom -@external(erlang, "erlang", "is_atom") -fn erlang_is_atom(value: dynamic.Dynamic) -> Bool - -/// Convert atom to string for comparison -@external(erlang, "erlang", "atom_to_binary") -fn erlang_atom_to_binary(atom: dynamic.Dynamic) -> BitArray - -/// Coerce a dynamic value known to be a DbHandle NIF resource. -/// Safe because the NIF guarantees the value inside {ok, Handle} -/// is always a valid DbHandle resource reference. -@external(erlang, "erlang", "identity") -fn coerce_to_db_handle(value: dynamic.Dynamic) -> nif_ffi.DbHandle - -/// Coerce a dynamic value known to be a TxnHandle NIF resource. -/// Safe because the NIF guarantees the value inside {ok, Handle} -/// is always a valid TxnHandle resource reference. -@external(erlang, "erlang", "identity") -fn coerce_to_txn_handle(value: dynamic.Dynamic) -> nif_ffi.TxnHandle - -/// Coerce a dynamic value known to be a BitArray (binary). -/// Safe because the NIF returns binaries for schema/journal/apply results -/// and we only call this after verifying the {ok, Value} tuple structure. -@external(erlang, "erlang", "identity") -fn coerce_to_bit_array(value: dynamic.Dynamic) -> BitArray - -/// Decode an Erlang {ok, Value} or {error, Reason} tuple. -/// Returns Ok(Value) for {ok, Value}, Error(reason_string) for {error, ...}. -/// This replaces the broken is_ok_result which accepted ANY atom as "ok". -fn decode_ok_result( - result: dynamic.Dynamic, -) -> Result(dynamic.Dynamic, String) { - case erlang_is_atom(result) { - True -> { - // Bare atom result (e.g. just 'ok' without a tuple) - let atom_bin = erlang_atom_to_binary(result) - case bit_array.to_string(atom_bin) { - Ok("ok") -> Ok(result) - Ok(other) -> Error(other) - Error(_) -> Error("unknown_atom") - } - } - False -> { - // Should be a tuple - let size = erlang_tuple_size(result) - case size >= 2 { - True -> { - let tag = erlang_element(1, result) - let tag_bin = erlang_atom_to_binary(tag) - case bit_array.to_string(tag_bin) { - Ok("ok") -> Ok(erlang_element(2, result)) - Ok("error") -> { - // Extract error reason - let reason = erlang_element(2, result) - case erlang_is_atom(reason) { - True -> { - let reason_bin = erlang_atom_to_binary(reason) - case bit_array.to_string(reason_bin) { - Ok(reason_str) -> Error(reason_str) - Error(_) -> Error("unknown_error") - } - } - False -> Error("nif_error") - } - } - _ -> Error("unexpected_nif_result") - } - } - False -> Error("malformed_nif_result") - } - } - } -} - -/// Decode a result that is either bare atom 'ok' or {error, Reason}. -/// Used for db_close, txn_commit, txn_abort which return atoms directly. -fn decode_atom_or_error( - result: dynamic.Dynamic, -) -> Result(Nil, String) { - case decode_ok_result(result) { - Ok(_) -> Ok(Nil) - Error(reason) -> Error(reason) - } -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/database.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/database.gleam deleted file mode 100644 index e81940b3..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/database.gleam +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith database connection manager - -import lithoglyph/client.{type Connection, ReadOnly, ReadWrite} -import lithoglyph/operations.{type Operation, encode_operation} -import gleam/option.{type Option, None, Some} - -/// Database state -pub opaque type DatabaseState { - DatabaseState(conn: Option(Connection), path: String) -} - -/// Create a new database state with the given path -pub fn new(path: String) -> DatabaseState { - DatabaseState(conn: None, path: path) -} - -/// Open the database connection -pub fn open(state: DatabaseState) -> #(DatabaseState, client.LithResult(Nil)) { - case state.conn { - Some(_) -> #(state, Ok(Nil)) - None -> { - case client.connect(state.path) { - Ok(conn) -> #(DatabaseState(..state, conn: Some(conn)), Ok(Nil)) - Error(e) -> #(state, Error(e)) - } - } - } -} - -/// Close the database connection -pub fn close(state: DatabaseState) -> #(DatabaseState, client.LithResult(Nil)) { - case state.conn { - None -> #(state, Ok(Nil)) - Some(conn) -> { - case client.disconnect(conn) { - Ok(_) -> #(DatabaseState(..state, conn: None), Ok(Nil)) - Error(e) -> #(state, Error(e)) - } - } - } -} - -/// Execute an operation on the database -pub fn execute( - state: DatabaseState, - op: Operation, -) -> #(DatabaseState, client.LithResult(BitArray)) { - case state.conn { - None -> #(state, Error(client.ConnectionError("Database not connected"))) - Some(conn) -> { - let encoded_op = encode_operation(op) - // Determine if this is a read-only or read-write operation - let mode = get_operation_mode(op) - case client.with_transaction(conn, mode, fn(txn) { - case client.apply_operation(txn, encoded_op) { - Ok(#(result, _provenance)) -> Ok(result) - Error(e) -> Error(e) - } - }) { - Ok(result) -> #(state, Ok(result)) - Error(e) -> #(state, Error(e)) - } - } - } -} - -/// Get the transaction mode for an operation -fn get_operation_mode(op: Operation) -> client.TransactionMode { - case op { - operations.CreateBase(_, _, _) -> ReadWrite - operations.GetBase(_) -> ReadOnly - operations.ListBases -> ReadOnly - operations.UpdateBase(_, _, _) -> ReadWrite - operations.DeleteBase(_) -> ReadWrite - operations.CreateTable(_, _, _, _, _) -> ReadWrite - operations.GetTable(_, _) -> ReadOnly - operations.ListTables(_) -> ReadOnly - operations.UpdateTable(_, _, _) -> ReadWrite - operations.DeleteTable(_, _) -> ReadWrite - operations.CreateRow(_, _, _, _) -> ReadWrite - operations.GetRow(_, _, _) -> ReadOnly - operations.ListRows(_, _, _, _, _) -> ReadOnly - operations.UpdateRow(_, _, _, _, _) -> ReadWrite - operations.DeleteRow(_, _, _) -> ReadWrite - operations.GetCell(_, _, _, _) -> ReadOnly - operations.UpdateCell(_, _, _, _, _, _) -> ReadWrite - operations.GetProvenance(_, _, _, _) -> ReadOnly - } -} - -/// Get the current connection (for advanced use cases) -pub fn get_connection(state: DatabaseState) -> Option(Connection) { - state.conn -} - -// ============================================================ -// Simplified Interface (for request handling) -// ============================================================ - -/// Execute a read-only operation on an existing connection -pub fn read(conn: Connection, op: Operation) -> client.LithResult(BitArray) { - let encoded_op = encode_operation(op) - client.with_transaction(conn, ReadOnly, fn(txn) { - case client.apply_operation(txn, encoded_op) { - Ok(#(result, _)) -> Ok(result) - Error(e) -> Error(e) - } - }) -} - -/// Execute a read-write operation on an existing connection -pub fn write(conn: Connection, op: Operation) -> client.LithResult(BitArray) { - let encoded_op = encode_operation(op) - client.with_transaction(conn, ReadWrite, fn(txn) { - case client.apply_operation(txn, encoded_op) { - Ok(#(result, _)) -> Ok(result) - Error(e) -> Error(e) - } - }) -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/gql.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/gql.gleam deleted file mode 100644 index 5473b94f..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/gql.gleam +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// GQL (Lith Query Language) builder for type-safe queries - -import gleam/list -import gleam/option.{type Option, None, Some} -import gleam/string - -/// GQL query builder -pub opaque type Query { - Query( - collection: String, - select_fields: List(String), - where_clause: Option(String), - order_by: Option(#(String, Order)), - limit_count: Option(Int), - offset_count: Option(Int), - ) -} - -pub type Order { - Asc - Desc -} - -/// Start building a query for a collection -pub fn from(collection: String) -> Query { - Query( - collection: collection, - select_fields: [], - where_clause: None, - order_by: None, - limit_count: None, - offset_count: None, - ) -} - -/// Select specific fields (empty = all fields) -pub fn select(query: Query, fields: List(String)) -> Query { - Query(..query, select_fields: fields) -} - -/// Add a where clause (GQL filter expression) -pub fn where(query: Query, clause: String) -> Query { - Query(..query, where_clause: Some(clause)) -} - -/// Add ordering -pub fn order_by(query: Query, field: String, order: Order) -> Query { - Query(..query, order_by: Some(#(field, order))) -} - -/// Limit results -pub fn limit(query: Query, count: Int) -> Query { - Query(..query, limit_count: Some(count)) -} - -/// Skip results -pub fn offset(query: Query, count: Int) -> Query { - Query(..query, offset_count: Some(count)) -} - -/// Build the GQL string -pub fn build(query: Query) -> String { - let select_part = case query.select_fields { - [] -> "*" - fields -> string.join(fields, ", ") - } - - let base = "SELECT " <> select_part <> " FROM " <> query.collection - - let with_where = case query.where_clause { - None -> base - Some(clause) -> base <> " WHERE " <> clause - } - - let with_order = case query.order_by { - None -> with_where - Some(#(field, Asc)) -> with_where <> " ORDER BY " <> field <> " ASC" - Some(#(field, Desc)) -> with_where <> " ORDER BY " <> field <> " DESC" - } - - let with_limit = case query.limit_count { - None -> with_order - Some(count) -> with_order <> " LIMIT " <> string.inspect(count) - } - - case query.offset_count { - None -> with_limit - Some(count) -> with_limit <> " OFFSET " <> string.inspect(count) - } -} - -// Mutation builders - -pub type Insert { - Insert(collection: String, fields: List(#(String, String))) -} - -pub type Update { - Update( - collection: String, - document_id: String, - fields: List(#(String, String)), - ) -} - -pub type Delete { - Delete(collection: String, document_id: String) -} - -/// Build an INSERT statement -pub fn insert(collection: String, fields: List(#(String, String))) -> String { - let field_names = list.map(fields, fn(f) { f.0 }) - let field_values = list.map(fields, fn(f) { "'" <> f.1 <> "'" }) - - "INSERT INTO " - <> collection - <> " (" - <> string.join(field_names, ", ") - <> ") VALUES (" - <> string.join(field_values, ", ") - <> ")" -} - -/// Build an UPDATE statement -pub fn update( - collection: String, - document_id: String, - fields: List(#(String, String)), -) -> String { - let set_clauses = - list.map(fields, fn(f) { f.0 <> " = '" <> f.1 <> "'" }) - |> string.join(", ") - - "UPDATE " <> collection <> " SET " <> set_clauses <> " WHERE _id = '" <> document_id <> "'" -} - -/// Build a DELETE statement -pub fn delete(collection: String, document_id: String) -> String { - "DELETE FROM " <> collection <> " WHERE _id = '" <> document_id <> "'" -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/nif_ffi.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/nif_ffi.gleam deleted file mode 100644 index fd709df1..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/nif_ffi.gleam +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// Lith NIF FFI - Raw Erlang NIF bindings -// -// All functions that return {ok, Value} or {error, Reason} from the Zig NIF -// are declared as returning Dynamic. The client module decodes these safely. -// NEVER call these directly — use lithoglyph/client instead. - -import gleam/dynamic - -/// NIF version function — returns {Major, Minor, Patch} tuple directly -@external(erlang, "lith_nif", "version") -pub fn nif_version() -> #(Int, Int, Int) - -/// NIF db_open — returns {ok, DbResource} or {error, Atom} or {error, Atom, Binary} -@external(erlang, "lith_nif", "db_open") -pub fn nif_db_open(path: BitArray) -> dynamic.Dynamic - -/// NIF db_close — returns atom 'ok' or {error, Atom} -@external(erlang, "lith_nif", "db_close") -pub fn nif_db_close(db: DbHandle) -> dynamic.Dynamic - -/// NIF txn_begin — returns {ok, TxnResource} or {error, Atom} -@external(erlang, "lith_nif", "txn_begin") -pub fn nif_txn_begin(db: DbHandle, mode: BitArray) -> dynamic.Dynamic - -/// NIF txn_commit — returns atom 'ok' or {error, Atom} -@external(erlang, "lith_nif", "txn_commit") -pub fn nif_txn_commit(txn: TxnHandle) -> dynamic.Dynamic - -/// NIF txn_abort — returns atom 'ok' -@external(erlang, "lith_nif", "txn_abort") -pub fn nif_txn_abort(txn: TxnHandle) -> dynamic.Dynamic - -/// NIF apply — returns {ok, Binary} or {error, Atom} -@external(erlang, "lith_nif", "apply") -pub fn nif_apply(txn: TxnHandle, op_cbor: BitArray) -> dynamic.Dynamic - -/// NIF schema — returns {ok, Binary} or {error, Atom} -@external(erlang, "lith_nif", "schema") -pub fn nif_schema(db: DbHandle) -> dynamic.Dynamic - -/// NIF journal — returns {ok, Binary} or {error, Atom} -@external(erlang, "lith_nif", "journal") -pub fn nif_journal(db: DbHandle, since: Int) -> dynamic.Dynamic - -// Opaque types for NIF resource handles (Erlang NIF resources) -pub type DbHandle - -pub type TxnHandle diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/operations.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/operations.gleam deleted file mode 100644 index 07db3adb..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/operations.gleam +++ /dev/null @@ -1,473 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lith operation types and CBOR encoding - -import lithoglyph/cbor -import gleam/list -import gleam/option.{type Option} -import types.{ - type CellValue, type Field, type FieldType, Attachment, AttachmentValue, - Barcode, Checkbox, CheckboxValue, Date, DateTime, DateValue, Email, Formula, - Link, LinkValue, Lookup, MultiSelect, MultiSelectValue, NullValue, Number, - NumberValue, Phone, Rating, Rollup, Select, SelectValue, Text, TextValue, Url, -} - -/// Lith operations -pub type Operation { - // Base operations - CreateBase(id: String, name: String, description: Option(String)) - GetBase(id: String) - ListBases - UpdateBase(id: String, name: Option(String), description: Option(String)) - DeleteBase(id: String) - // Table operations - CreateTable( - base_id: String, - id: String, - name: String, - fields: List(Field), - primary_field_id: String, - ) - GetTable(base_id: String, table_id: String) - ListTables(base_id: String) - UpdateTable(base_id: String, table_id: String, name: Option(String)) - DeleteTable(base_id: String, table_id: String) - // Row operations - CreateRow(base_id: String, table_id: String, id: String, cells: List(CellData)) - GetRow(base_id: String, table_id: String, row_id: String) - ListRows( - base_id: String, - table_id: String, - limit: Option(Int), - offset: Option(Int), - filter: Option(String), - ) - UpdateRow( - base_id: String, - table_id: String, - row_id: String, - cells: List(CellData), - rationale: Option(String), - ) - DeleteRow(base_id: String, table_id: String, row_id: String) - // Cell operations - GetCell(base_id: String, table_id: String, row_id: String, field_id: String) - UpdateCell( - base_id: String, - table_id: String, - row_id: String, - field_id: String, - value: CellValue, - rationale: Option(String), - ) - // Provenance - GetProvenance( - base_id: String, - table_id: String, - row_id: String, - field_id: String, - ) -} - -/// Cell data for row operations -pub type CellData { - CellData(field_id: String, value: CellValue) -} - -/// Encode an operation to CBOR -pub fn encode_operation(op: Operation) -> BitArray { - case op { - CreateBase(id, name, desc) -> encode_create_base(id, name, desc) - GetBase(id) -> encode_get_base(id) - ListBases -> encode_list_bases() - UpdateBase(id, name, desc) -> encode_update_base(id, name, desc) - DeleteBase(id) -> encode_delete_base(id) - CreateTable(base_id, id, name, fields, primary) -> - encode_create_table(base_id, id, name, fields, primary) - GetTable(base_id, table_id) -> encode_get_table(base_id, table_id) - ListTables(base_id) -> encode_list_tables(base_id) - UpdateTable(base_id, table_id, name) -> - encode_update_table(base_id, table_id, name) - DeleteTable(base_id, table_id) -> encode_delete_table(base_id, table_id) - CreateRow(base_id, table_id, id, cells) -> - encode_create_row(base_id, table_id, id, cells) - GetRow(base_id, table_id, row_id) -> - encode_get_row(base_id, table_id, row_id) - ListRows(base_id, table_id, limit, offset, filter) -> - encode_list_rows(base_id, table_id, limit, offset, filter) - UpdateRow(base_id, table_id, row_id, cells, rationale) -> - encode_update_row(base_id, table_id, row_id, cells, rationale) - DeleteRow(base_id, table_id, row_id) -> - encode_delete_row(base_id, table_id, row_id) - GetCell(base_id, table_id, row_id, field_id) -> - encode_get_cell(base_id, table_id, row_id, field_id) - UpdateCell(base_id, table_id, row_id, field_id, value, rationale) -> - encode_update_cell(base_id, table_id, row_id, field_id, value, rationale) - GetProvenance(base_id, table_id, row_id, field_id) -> - encode_get_provenance(base_id, table_id, row_id, field_id) - } -} - -// ============================================================ -// Operation Encoders -// ============================================================ - -fn encode_create_base( - id: String, - name: String, - desc: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("create_base")), - #(cbor.encode_text("id"), cbor.encode_text(id)), - #(cbor.encode_text("name"), cbor.encode_text(name)), - #(cbor.encode_text("description"), cbor.encode_optional(desc, cbor.encode_text)), - ]) -} - -fn encode_get_base(id: String) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("get_base")), - #(cbor.encode_text("id"), cbor.encode_text(id)), - ]) -} - -fn encode_list_bases() -> BitArray { - cbor.encode_map([#(cbor.encode_text("op"), cbor.encode_text("list_bases"))]) -} - -fn encode_update_base( - id: String, - name: Option(String), - desc: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("update_base")), - #(cbor.encode_text("id"), cbor.encode_text(id)), - #(cbor.encode_text("name"), cbor.encode_optional(name, cbor.encode_text)), - #(cbor.encode_text("description"), cbor.encode_optional(desc, cbor.encode_text)), - ]) -} - -fn encode_delete_base(id: String) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("delete_base")), - #(cbor.encode_text("id"), cbor.encode_text(id)), - ]) -} - -fn encode_create_table( - base_id: String, - id: String, - name: String, - fields: List(Field), - primary_field_id: String, -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("create_table")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("id"), cbor.encode_text(id)), - #(cbor.encode_text("name"), cbor.encode_text(name)), - #(cbor.encode_text("fields"), encode_fields(fields)), - #(cbor.encode_text("primary_field_id"), cbor.encode_text(primary_field_id)), - ]) -} - -fn encode_get_table(base_id: String, table_id: String) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("get_table")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - ]) -} - -fn encode_list_tables(base_id: String) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("list_tables")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - ]) -} - -fn encode_update_table( - base_id: String, - table_id: String, - name: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("update_table")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("name"), cbor.encode_optional(name, cbor.encode_text)), - ]) -} - -fn encode_delete_table(base_id: String, table_id: String) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("delete_table")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - ]) -} - -fn encode_create_row( - base_id: String, - table_id: String, - id: String, - cells: List(CellData), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("create_row")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("id"), cbor.encode_text(id)), - #(cbor.encode_text("cells"), encode_cells(cells)), - ]) -} - -fn encode_get_row( - base_id: String, - table_id: String, - row_id: String, -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("get_row")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - ]) -} - -fn encode_list_rows( - base_id: String, - table_id: String, - limit: Option(Int), - offset: Option(Int), - filter: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("list_rows")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("limit"), cbor.encode_optional(limit, cbor.encode_int)), - #(cbor.encode_text("offset"), cbor.encode_optional(offset, cbor.encode_int)), - #(cbor.encode_text("filter"), cbor.encode_optional(filter, cbor.encode_text)), - ]) -} - -fn encode_update_row( - base_id: String, - table_id: String, - row_id: String, - cells: List(CellData), - rationale: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("update_row")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - #(cbor.encode_text("cells"), encode_cells(cells)), - #(cbor.encode_text("rationale"), cbor.encode_optional(rationale, cbor.encode_text)), - ]) -} - -fn encode_delete_row( - base_id: String, - table_id: String, - row_id: String, -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("delete_row")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - ]) -} - -fn encode_get_cell( - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("get_cell")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - #(cbor.encode_text("field_id"), cbor.encode_text(field_id)), - ]) -} - -fn encode_update_cell( - base_id: String, - table_id: String, - row_id: String, - field_id: String, - value: CellValue, - rationale: Option(String), -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("update_cell")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - #(cbor.encode_text("field_id"), cbor.encode_text(field_id)), - #(cbor.encode_text("value"), encode_cell_value(value)), - #(cbor.encode_text("rationale"), cbor.encode_optional(rationale, cbor.encode_text)), - ]) -} - -fn encode_get_provenance( - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> BitArray { - cbor.encode_map([ - #(cbor.encode_text("op"), cbor.encode_text("get_provenance")), - #(cbor.encode_text("base_id"), cbor.encode_text(base_id)), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - #(cbor.encode_text("row_id"), cbor.encode_text(row_id)), - #(cbor.encode_text("field_id"), cbor.encode_text(field_id)), - ]) -} - -// ============================================================ -// Helper Encoders -// ============================================================ - -fn encode_cell_value(value: CellValue) -> BitArray { - case value { - TextValue(s) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("text")), - #(cbor.encode_text("value"), cbor.encode_text(s)), - ]) - NumberValue(n) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("number")), - #(cbor.encode_text("value"), cbor.encode_float(n)), - ]) - SelectValue(s) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("select")), - #(cbor.encode_text("value"), cbor.encode_text(s)), - ]) - MultiSelectValue(items) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("multi_select")), - #( - cbor.encode_text("value"), - cbor.encode_array(list.map(items, cbor.encode_text)), - ), - ]) - DateValue(s) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("date")), - #(cbor.encode_text("value"), cbor.encode_text(s)), - ]) - CheckboxValue(b) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("checkbox")), - #(cbor.encode_text("value"), cbor.encode_bool(b)), - ]) - LinkValue(ids) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("link")), - #( - cbor.encode_text("value"), - cbor.encode_array(list.map(ids, cbor.encode_text)), - ), - ]) - AttachmentValue(ids) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("attachment")), - #( - cbor.encode_text("value"), - cbor.encode_array(list.map(ids, cbor.encode_text)), - ), - ]) - NullValue -> cbor.encode_null() - } -} - -fn encode_cells(cells: List(CellData)) -> BitArray { - cbor.encode_array(list.map(cells, encode_cell_data)) -} - -fn encode_cell_data(cell: CellData) -> BitArray { - let CellData(field_id, value) = cell - cbor.encode_map([ - #(cbor.encode_text("field_id"), cbor.encode_text(field_id)), - #(cbor.encode_text("value"), encode_cell_value(value)), - ]) -} - -fn encode_fields(fields: List(Field)) -> BitArray { - cbor.encode_array(list.map(fields, encode_field)) -} - -fn encode_field(field: Field) -> BitArray { - let types.Field(id, name, field_type, required, default_value) = field - cbor.encode_map([ - #(cbor.encode_text("id"), cbor.encode_text(id)), - #(cbor.encode_text("name"), cbor.encode_text(name)), - #(cbor.encode_text("field_type"), encode_field_type(field_type)), - #(cbor.encode_text("required"), cbor.encode_bool(required)), - #(cbor.encode_text("default_value"), cbor.encode_optional(default_value, cbor.encode_text)), - ]) -} - -fn encode_field_type(ft: FieldType) -> BitArray { - case ft { - Text -> cbor.encode_text("text") - Number -> cbor.encode_text("number") - Select(opts) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("select")), - #( - cbor.encode_text("options"), - cbor.encode_array(list.map(opts, cbor.encode_text)), - ), - ]) - MultiSelect(opts) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("multi_select")), - #( - cbor.encode_text("options"), - cbor.encode_array(list.map(opts, cbor.encode_text)), - ), - ]) - Date -> cbor.encode_text("date") - DateTime -> cbor.encode_text("datetime") - Checkbox -> cbor.encode_text("checkbox") - Link(table_id) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("link")), - #(cbor.encode_text("table_id"), cbor.encode_text(table_id)), - ]) - Attachment -> cbor.encode_text("attachment") - Formula(expr) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("formula")), - #(cbor.encode_text("expression"), cbor.encode_text(expr)), - ]) - Rollup(linked, agg) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("rollup")), - #(cbor.encode_text("linked_field"), cbor.encode_text(linked)), - #(cbor.encode_text("aggregation"), cbor.encode_text(agg)), - ]) - Lookup(linked, lookup) -> - cbor.encode_map([ - #(cbor.encode_text("type"), cbor.encode_text("lookup")), - #(cbor.encode_text("linked_field"), cbor.encode_text(linked)), - #(cbor.encode_text("lookup_field"), cbor.encode_text(lookup)), - ]) - Url -> cbor.encode_text("url") - Email -> cbor.encode_text("email") - Phone -> cbor.encode_text("phone") - Rating -> cbor.encode_text("rating") - Barcode -> cbor.encode_text("barcode") - } -} diff --git a/lithoglyph/glyphbase/server/src/lithoglyph/safety.gleam b/lithoglyph/glyphbase/server/src/lithoglyph/safety.gleam deleted file mode 100644 index 825f5aca..00000000 --- a/lithoglyph/glyphbase/server/src/lithoglyph/safety.gleam +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// safety.gleam - Proven-compatible safety functions for Glyphbase -// -// Pure Gleam implementations of critical safety checks from the proven -// library (SafePath, SafeString). These match the proven API signatures -// so they can be replaced with real NIF calls when the proven build -// pipeline (Idris2 → RefC → Zig → NIF) is operational. -// -// TODO: Replace with `import proven/path`, `import proven/string_ops` -// once libproven_nif.so is built. - -import gleam/string - -// ============================================================ -// SafePath — Directory traversal prevention -// ============================================================ - -/// Check if a path contains directory traversal sequences. -/// Matches proven/path.has_traversal/1 API. -pub fn path_has_traversal(path: String) -> Result(Bool, String) { - case string.contains(path, "..") { - True -> Ok(True) - False -> - case string.contains(path, "\u{0000}") { - True -> Ok(True) - False -> Ok(False) - } - } -} - -/// Sanitize a filename by removing path separators and traversal sequences. -/// Matches proven/path.sanitize_filename/1 API. -pub fn sanitize_filename(filename: String) -> Result(String, String) { - let cleaned = - filename - |> string.replace("..", "") - |> string.replace("/", "_") - |> string.replace("\\", "_") - |> string.replace("\u{0000}", "") - - case string.is_empty(cleaned) { - True -> Error("filename_empty_after_sanitization") - False -> Ok(cleaned) - } -} - -// ============================================================ -// SafeString — Escaping for injection prevention -// ============================================================ - -/// Escape a string for safe use in SQL (single-quote doubling). -/// Matches proven/string_ops.escape_sql/1 API. -pub fn escape_sql(value: String) -> Result(String, String) { - Ok(string.replace(value, "'", "''")) -} - -/// Escape a string for safe use in HTML content. -/// Matches proven/string_ops.escape_html/1 API. -pub fn escape_html(value: String) -> Result(String, String) { - value - |> string.replace("&", "&") - |> string.replace("<", "<") - |> string.replace(">", ">") - |> string.replace("\"", """) - |> string.replace("'", "'") - |> Ok -} - -/// Escape a string for safe use in JavaScript string literals. -/// Matches proven/string_ops.escape_js/1 API. -pub fn escape_js(value: String) -> Result(String, String) { - value - |> string.replace("\\", "\\\\") - |> string.replace("\"", "\\\"") - |> string.replace("'", "\\'") - |> string.replace("<", "\\u003C") - |> string.replace(">", "\\u003E") - |> string.replace("/", "\\/") - |> string.replace("\n", "\\n") - |> string.replace("\r", "\\r") - |> Ok -} diff --git a/lithoglyph/glyphbase/server/src/router.gleam b/lithoglyph/glyphbase/server/src/router.gleam deleted file mode 100644 index fcfdd32e..00000000 --- a/lithoglyph/glyphbase/server/src/router.gleam +++ /dev/null @@ -1,595 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// HTTP router for Glyphbase API - -import lithoglyph/client.{type Connection} -import lithoglyph/database -import lithoglyph/operations -import gleam/dynamic/decode -import gleam/http.{Delete, Get, Patch, Post} -import gleam/int -import gleam/json -import gleam/list -import gleam/option.{type Option, None} -import gleam/result -import types.{TextValue} -import wisp.{type Request, type Response} - -/// Context passed to handlers -pub type Context { - Context(db: Connection) -} - -/// Main router for all API endpoints -pub fn handle_request(req: Request, ctx: Context) -> Response { - case wisp.path_segments(req) { - // Health check - [] -> home(req) - ["health"] -> health_check(req) - - // Base CRUD - ["api", "bases"] -> bases_handler(req, ctx) - ["api", "bases", base_id] -> base_handler(req, ctx, base_id) - - // Table CRUD - ["api", "bases", base_id, "tables"] -> tables_handler(req, ctx, base_id) - ["api", "bases", base_id, "tables", table_id] -> - table_handler(req, ctx, base_id, table_id) - - // Row CRUD - ["api", "bases", base_id, "tables", table_id, "rows"] -> - rows_handler(req, ctx, base_id, table_id) - ["api", "bases", base_id, "tables", table_id, "rows", row_id] -> - row_handler(req, ctx, base_id, table_id, row_id) - - // Cell operations - ["api", "bases", base_id, "tables", table_id, "rows", row_id, "cells", field_id] -> - cell_handler(req, ctx, base_id, table_id, row_id, field_id) - - // Provenance - ["api", "bases", base_id, "tables", table_id, "rows", row_id, "cells", field_id, "provenance"] -> - provenance_handler(req, ctx, base_id, table_id, row_id, field_id) - - // Views - ["api", "bases", base_id, "tables", table_id, "views"] -> - views_handler(req, ctx, base_id, table_id) - ["api", "bases", base_id, "tables", table_id, "views", view_id] -> - view_handler(req, ctx, base_id, table_id, view_id) - - _ -> wisp.not_found() - } -} - -fn json_response(body: json.Json, status: Int) -> Response { - let json_string = json.to_string(body) - wisp.json_response(json_string, status) -} - -fn home(_req: Request) -> Response { - json_response( - json.object([ - #("name", json.string("Glyphbase")), - #("version", json.string("0.1.0")), - ]), - 200, - ) -} - -fn health_check(_req: Request) -> Response { - json_response(json.object([#("status", json.string("ok"))]), 200) -} - -// ============================================================ -// Base handlers -// ============================================================ - -fn bases_handler(req: Request, ctx: Context) -> Response { - case req.method { - Get -> list_bases(ctx) - Post -> create_base(req, ctx) - _ -> wisp.method_not_allowed([Get, Post]) - } -} - -fn base_handler(req: Request, ctx: Context, base_id: String) -> Response { - case req.method { - Get -> get_base(ctx, base_id) - Patch -> update_base(req, ctx, base_id) - Delete -> delete_base(ctx, base_id) - _ -> wisp.method_not_allowed([Get, Patch, Delete]) - } -} - -fn list_bases(ctx: Context) -> Response { - let op = operations.ListBases - case database.read(ctx.db, op) { - Ok(_) -> json_response(json.object([#("bases", json.array([], json.string))]), 200) - Error(e) -> error_response(e) - } -} - -fn create_base(req: Request, ctx: Context) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use id <- decode.field("id", decode.string) - use name <- decode.field("name", decode.string) - use desc <- decode.optional_field("description", None, decode.optional(decode.string)) - decode.success(#(id, name, desc)) - } - - case decode.run(body, decoder) { - Ok(#(id, name, description)) -> { - let op = operations.CreateBase(id, name, description) - case database.write(ctx.db, op) { - Ok(_) -> - json_response( - json.object([#("id", json.string(id)), #("name", json.string(name))]), - 201, - ) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn get_base(ctx: Context, base_id: String) -> Response { - let op = operations.GetBase(base_id) - case database.read(ctx.db, op) { - Ok(_) -> - json_response( - json.object([ - #("id", json.string(base_id)), - #("name", json.string("Demo Base")), - #("tables", json.array([], json.string)), - ]), - 200, - ) - Error(e) -> error_response(e) - } -} - -fn update_base(req: Request, ctx: Context, base_id: String) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use name <- decode.optional_field("name", None, decode.optional(decode.string)) - use desc <- decode.optional_field("description", None, decode.optional(decode.string)) - decode.success(#(name, desc)) - } - - case decode.run(body, decoder) { - Ok(#(name, description)) -> { - let op = operations.UpdateBase(base_id, name, description) - case database.write(ctx.db, op) { - Ok(_) -> json_response(json.object([#("updated", json.bool(True))]), 200) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn delete_base(ctx: Context, base_id: String) -> Response { - let op = operations.DeleteBase(base_id) - case database.write(ctx.db, op) { - Ok(_) -> wisp.no_content() - Error(e) -> error_response(e) - } -} - -// ============================================================ -// Table handlers -// ============================================================ - -fn tables_handler(req: Request, ctx: Context, base_id: String) -> Response { - case req.method { - Get -> list_tables(ctx, base_id) - Post -> create_table(req, ctx, base_id) - _ -> wisp.method_not_allowed([Get, Post]) - } -} - -fn table_handler( - req: Request, - ctx: Context, - base_id: String, - table_id: String, -) -> Response { - case req.method { - Get -> get_table(ctx, base_id, table_id) - Patch -> update_table(req, ctx, base_id, table_id) - Delete -> delete_table(ctx, base_id, table_id) - _ -> wisp.method_not_allowed([Get, Patch, Delete]) - } -} - -fn list_tables(ctx: Context, base_id: String) -> Response { - let op = operations.ListTables(base_id) - case database.read(ctx.db, op) { - Ok(_) -> json_response(json.object([#("tables", json.array([], json.string))]), 200) - Error(e) -> error_response(e) - } -} - -fn create_table(req: Request, ctx: Context, base_id: String) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use id <- decode.field("id", decode.string) - use name <- decode.field("name", decode.string) - use primary <- decode.field("primaryFieldId", decode.string) - decode.success(#(id, name, primary)) - } - - case decode.run(body, decoder) { - Ok(#(id, name, primary_field_id)) -> { - let op = operations.CreateTable(base_id, id, name, [], primary_field_id) - case database.write(ctx.db, op) { - Ok(_) -> - json_response( - json.object([#("id", json.string(id)), #("name", json.string(name))]), - 201, - ) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn get_table(ctx: Context, base_id: String, table_id: String) -> Response { - let op = operations.GetTable(base_id, table_id) - case database.read(ctx.db, op) { - Ok(_) -> - json_response( - json.object([ - #("id", json.string(table_id)), - #("name", json.string("Demo Table")), - #("fields", json.array([], json.string)), - ]), - 200, - ) - Error(e) -> error_response(e) - } -} - -fn update_table( - req: Request, - ctx: Context, - base_id: String, - table_id: String, -) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use name <- decode.optional_field("name", None, decode.optional(decode.string)) - decode.success(name) - } - - case decode.run(body, decoder) { - Ok(name) -> { - let op = operations.UpdateTable(base_id, table_id, name) - case database.write(ctx.db, op) { - Ok(_) -> json_response(json.object([#("updated", json.bool(True))]), 200) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn delete_table(ctx: Context, base_id: String, table_id: String) -> Response { - let op = operations.DeleteTable(base_id, table_id) - case database.write(ctx.db, op) { - Ok(_) -> wisp.no_content() - Error(e) -> error_response(e) - } -} - -// ============================================================ -// Row handlers -// ============================================================ - -fn rows_handler( - req: Request, - ctx: Context, - base_id: String, - table_id: String, -) -> Response { - case req.method { - Get -> list_rows(ctx, base_id, table_id, req) - Post -> create_row(req, ctx, base_id, table_id) - _ -> wisp.method_not_allowed([Get, Post]) - } -} - -fn row_handler( - req: Request, - ctx: Context, - base_id: String, - table_id: String, - row_id: String, -) -> Response { - case req.method { - Get -> get_row(ctx, base_id, table_id, row_id) - Patch -> update_row(req, ctx, base_id, table_id, row_id) - Delete -> delete_row(ctx, base_id, table_id, row_id) - _ -> wisp.method_not_allowed([Get, Patch, Delete]) - } -} - -fn list_rows( - ctx: Context, - base_id: String, - table_id: String, - req: Request, -) -> Response { - // Parse query params for pagination - let limit = get_query_int(req, "limit") - let offset = get_query_int(req, "offset") - let filter = get_query_string(req, "filter") - - let op = operations.ListRows(base_id, table_id, limit, offset, filter) - case database.read(ctx.db, op) { - Ok(_) -> json_response(json.object([#("rows", json.array([], json.string))]), 200) - Error(e) -> error_response(e) - } -} - -fn create_row( - req: Request, - ctx: Context, - base_id: String, - table_id: String, -) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use id <- decode.field("id", decode.string) - decode.success(id) - } - - case decode.run(body, decoder) { - Ok(id) -> { - let op = operations.CreateRow(base_id, table_id, id, []) - case database.write(ctx.db, op) { - Ok(_) -> json_response(json.object([#("id", json.string(id))]), 201) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn get_row( - ctx: Context, - base_id: String, - table_id: String, - row_id: String, -) -> Response { - let op = operations.GetRow(base_id, table_id, row_id) - case database.read(ctx.db, op) { - Ok(_) -> - json_response( - json.object([#("id", json.string(row_id)), #("cells", json.object([]))]), - 200, - ) - Error(e) -> error_response(e) - } -} - -fn update_row( - req: Request, - ctx: Context, - base_id: String, - table_id: String, - row_id: String, -) -> Response { - use body <- wisp.require_json(req) - - let decoder = { - use rationale <- decode.optional_field("rationale", None, decode.optional(decode.string)) - decode.success(rationale) - } - - case decode.run(body, decoder) { - Ok(rationale) -> { - let op = operations.UpdateRow(base_id, table_id, row_id, [], rationale) - case database.write(ctx.db, op) { - Ok(_) -> json_response(json.object([#("updated", json.bool(True))]), 200) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -fn delete_row( - ctx: Context, - base_id: String, - table_id: String, - row_id: String, -) -> Response { - let op = operations.DeleteRow(base_id, table_id, row_id) - case database.write(ctx.db, op) { - Ok(_) -> wisp.no_content() - Error(e) -> error_response(e) - } -} - -// ============================================================ -// Cell handlers -// ============================================================ - -fn cell_handler( - req: Request, - ctx: Context, - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> Response { - case req.method { - Get -> get_cell(ctx, base_id, table_id, row_id, field_id) - Patch -> update_cell(req, ctx, base_id, table_id, row_id, field_id) - _ -> wisp.method_not_allowed([Get, Patch]) - } -} - -fn get_cell( - ctx: Context, - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> Response { - let op = operations.GetCell(base_id, table_id, row_id, field_id) - case database.read(ctx.db, op) { - Ok(_) -> json_response(json.object([#("value", json.null())]), 200) - Error(e) -> error_response(e) - } -} - -fn update_cell( - req: Request, - ctx: Context, - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> Response { - use body <- wisp.require_json(req) - - // Simple decoder - just get the rationale - let decoder = { - use rationale <- decode.optional_field("rationale", None, decode.optional(decode.string)) - decode.success(rationale) - } - - case decode.run(body, decoder) { - Ok(rationale) -> { - // For now, use a default text value - proper parsing would need more work - let value = TextValue("") - let op = - operations.UpdateCell(base_id, table_id, row_id, field_id, value, rationale) - case database.write(ctx.db, op) { - Ok(_) -> json_response(json.object([#("updated", json.bool(True))]), 200) - Error(e) -> error_response(e) - } - } - Error(_) -> json_response(json.object([#("error", json.string("Invalid request body"))]), 400) - } -} - -// ============================================================ -// Provenance handler -// ============================================================ - -fn provenance_handler( - _req: Request, - ctx: Context, - base_id: String, - table_id: String, - row_id: String, - field_id: String, -) -> Response { - let op = operations.GetProvenance(base_id, table_id, row_id, field_id) - case database.read(ctx.db, op) { - Ok(_) -> json_response(json.object([#("entries", json.array([], json.string))]), 200) - Error(e) -> error_response(e) - } -} - -// ============================================================ -// View handlers (placeholder - views are complex) -// ============================================================ - -fn views_handler( - req: Request, - _ctx: Context, - _base_id: String, - _table_id: String, -) -> Response { - case req.method { - Get -> json_response(json.object([#("views", json.array([], json.string))]), 200) - Post -> json_response(json.object([#("id", json.string("view_new"))]), 201) - _ -> wisp.method_not_allowed([Get, Post]) - } -} - -fn view_handler( - req: Request, - _ctx: Context, - _base_id: String, - _table_id: String, - view_id: String, -) -> Response { - case req.method { - Get -> - json_response( - json.object([ - #("id", json.string(view_id)), - #("type", json.string("grid")), - ]), - 200, - ) - Patch -> json_response(json.object([#("updated", json.bool(True))]), 200) - Delete -> wisp.no_content() - _ -> wisp.method_not_allowed([Get, Patch, Delete]) - } -} - -// ============================================================ -// Helpers -// ============================================================ - -fn error_response(error: client.LithError) -> Response { - let message = case error { - client.ConnectionError(msg) -> "Connection error: " <> msg - client.TransactionError(msg) -> "Transaction error: " <> msg - client.QueryError(msg) -> "Query error: " <> msg - client.ValidationError(msg) -> "Validation error: " <> msg - client.ProvenanceError(msg) -> "Provenance error: " <> msg - client.NotFound(entity, id) -> entity <> " not found: " <> id - client.PermissionDenied(action) -> "Permission denied: " <> action - client.NifNotLoaded -> "Database NIF not loaded" - client.NifError(reason) -> "NIF error: " <> reason - client.ParseFailed -> "Failed to parse CBOR data" - client.InvalidHandle -> "Invalid database or transaction handle" - client.PathTraversal(path) -> "Path traversal rejected: " <> path - } - let status = case error { - client.PathTraversal(_) -> 400 - client.NotFound(_, _) -> 404 - client.PermissionDenied(_) -> 403 - client.ValidationError(_) -> 400 - _ -> 500 - } - json_response(json.object([#("error", json.string(message))]), status) -} - -fn get_query_int(req: Request, key: String) -> Option(Int) { - case wisp.get_query(req) { - [] -> None - params -> { - params - |> list.find(fn(p) { p.0 == key }) - |> result.map(fn(p) { p.1 }) - |> result.try(int.parse) - |> option.from_result - } - } -} - -fn get_query_string(req: Request, key: String) -> Option(String) { - case wisp.get_query(req) { - [] -> None - params -> { - params - |> list.find(fn(p) { p.0 == key }) - |> result.map(fn(p) { p.1 }) - |> option.from_result - } - } -} diff --git a/lithoglyph/glyphbase/server/src/types.gleam b/lithoglyph/glyphbase/server/src/types.gleam deleted file mode 100644 index 033f86a7..00000000 --- a/lithoglyph/glyphbase/server/src/types.gleam +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Core types for Glyphbase server - -import gleam/option.{type Option} - -/// Field types supported by Glyphbase -pub type FieldType { - Text - Number - Select(options: List(String)) - MultiSelect(options: List(String)) - Date - DateTime - Checkbox - Link(table_id: String) - Attachment - Formula(expression: String) - Rollup(linked_field: String, aggregation: String) - Lookup(linked_field: String, lookup_field: String) - Url - Email - Phone - Rating - Barcode -} - -/// Field configuration -pub type Field { - Field( - id: String, - name: String, - field_type: FieldType, - required: Bool, - default_value: Option(String), - ) -} - -/// Cell values -pub type CellValue { - TextValue(String) - NumberValue(Float) - SelectValue(String) - MultiSelectValue(List(String)) - DateValue(String) - CheckboxValue(Bool) - LinkValue(List(String)) - AttachmentValue(List(String)) - NullValue -} - -/// Provenance entry for a cell change -pub type ProvenanceEntry { - ProvenanceEntry( - timestamp: String, - user_id: String, - user_name: String, - previous_value: Option(CellValue), - new_value: CellValue, - rationale: Option(String), - ) -} - -/// A cell with its value and provenance history -pub type Cell { - Cell( - field_id: String, - value: CellValue, - provenance: List(ProvenanceEntry), - ) -} - -/// A row in a table -pub type Row { - Row( - id: String, - cells: List(Cell), - created_at: String, - updated_at: String, - ) -} - -/// A table definition -pub type Table { - Table( - id: String, - name: String, - fields: List(Field), - primary_field_id: String, - ) -} - -/// A base (database) -pub type Base { - Base( - id: String, - name: String, - description: Option(String), - tables: List(Table), - created_at: String, - updated_at: String, - ) -} - -/// View types -pub type ViewType { - Grid - Kanban(group_by_field: String) - Calendar(date_field: String) - Gallery(image_field: String) - Form -} - -/// View configuration -pub type View { - View( - id: String, - name: String, - table_id: String, - view_type: ViewType, - visible_fields: List(String), - sort_by: Option(#(String, SortDirection)), - filter_by: Option(String), - ) -} - -pub type SortDirection { - Asc - Desc -} diff --git a/lithoglyph/glyphbase/server/test/lith_client_test.gleam b/lithoglyph/glyphbase/server/test/lith_client_test.gleam deleted file mode 100644 index 5adb36a8..00000000 --- a/lithoglyph/glyphbase/server/test/lith_client_test.gleam +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Lithoglyph Client Integration Test - M10 PoC - -import lithoglyph/client -import gleam/io -import gleam/string -import gleeunit - -pub fn main() { - gleeunit.main() -} - -pub fn version_test() { - io.println("\n=== Lithoglyph Client NIF Test ===\n") - - // Test 1: Version - io.println("Test 1: Getting version...") - let version = client.version() - io.println(" ✓ Version: " <> string.inspect(version)) - - // Verify version is (1, 0, 0) - case version { - #(1, 0, 0) -> io.println(" ✓ Version matches expected v1.0.0\n") - _ -> panic as "Unexpected version" - } -} - -pub fn connection_test() { - // Test 2: Open connection - io.println("Test 2: Opening database connection...") - case client.connect("/tmp/lithoglyph_test") { - Ok(conn) -> { - io.println(" ✓ Connection opened\n") - - // Test 3: Get schema - io.println("Test 3: Getting schema...") - case client.get_schema(conn) { - Ok(schema) -> { - io.println(" ✓ Schema retrieved: " <> string.inspect(schema)) - io.println(" (CBOR empty map: <<160>>)\n") - } - Error(e) -> { - io.println(" ✗ Schema failed: " <> string.inspect(e)) - panic as "Schema retrieval failed" - } - } - - // Test 4: Get journal - io.println("Test 4: Getting journal...") - case client.get_journal(conn, 0) { - Ok(journal) -> { - io.println(" ✓ Journal retrieved: " <> string.inspect(journal)) - io.println(" (CBOR empty array: <<128>>)\n") - } - Error(e) -> { - io.println(" ✗ Journal failed: " <> string.inspect(e)) - panic as "Journal retrieval failed" - } - } - - // Test 5: Close connection - io.println("Test 5: Closing database connection...") - case client.disconnect(conn) { - Ok(_) -> io.println(" ✓ Connection closed\n") - Error(e) -> { - io.println(" ✗ Close failed: " <> string.inspect(e)) - panic as "Connection close failed" - } - } - } - Error(e) -> { - io.println(" ✗ Connection failed: " <> string.inspect(e)) - panic as "Connection open failed" - } - } -} - -pub fn transaction_test() { - io.println("Test 6: Transaction flow...") - - case client.connect("/tmp/lithoglyph_test") { - Ok(conn) -> { - // Test 6a: Begin transaction - io.println(" 6a: Beginning transaction...") - case client.begin_transaction(conn, client.ReadWrite) { - Ok(txn) -> { - io.println(" ✓ Transaction started\n") - - // Test 6b: Apply operation (CBOR map {1: 2}) - io.println(" 6b: Applying operation...") - let cbor_map = <<0xa1, 0x01, 0x02>> - case client.apply_operation(txn, cbor_map) { - Ok(#(block_id, _provenance)) -> { - io.println(" ✓ Operation applied") - io.println(" Block ID: " <> string.inspect(block_id)) - io.println(" (Expected: <<0,0,0,0,0,0,0,1>>)\n") - } - Error(e) -> { - io.println(" ✗ Apply failed: " <> string.inspect(e)) - let _ = client.abort(txn) - panic as "Apply operation failed" - } - } - - // Test 6c: Commit transaction - io.println(" 6c: Committing transaction...") - case client.commit(txn) { - Ok(_) -> io.println(" ✓ Transaction committed\n") - Error(e) -> { - io.println(" ✗ Commit failed: " <> string.inspect(e)) - panic as "Transaction commit failed" - } - } - } - Error(e) -> { - io.println(" ✗ Begin transaction failed: " <> string.inspect(e)) - panic as "Transaction begin failed" - } - } - - // Clean up - let _ = client.disconnect(conn) - } - Error(e) -> { - io.println(" ✗ Connection failed: " <> string.inspect(e)) - panic as "Connection failed" - } - } -} - -pub fn with_transaction_test() { - io.println("Test 7: High-level with_transaction...") - - case client.connect("/tmp/lithoglyph_test") { - Ok(conn) -> { - // Use the high-level with_transaction helper - case - client.with_transaction(conn, client.ReadWrite, fn(txn) { - let cbor_map = <<0xa1, 0x02, 0x03>> - client.apply_operation(txn, cbor_map) - }) - { - Ok(_) -> io.println(" ✓ with_transaction completed successfully\n") - Error(e) -> { - io.println(" ✗ with_transaction failed: " <> string.inspect(e)) - panic as "with_transaction failed" - } - } - - // Clean up - let _ = client.disconnect(conn) - io.println("=== All tests passed! ===\n") - } - Error(e) -> { - io.println(" ✗ Connection failed: " <> string.inspect(e)) - panic as "Connection failed" - } - } -} diff --git a/lithoglyph/glyphbase/server/test/lith_test.gleam.old b/lithoglyph/glyphbase/server/test/lith_test.gleam.old deleted file mode 100644 index c6070918..00000000 --- a/lithoglyph/glyphbase/server/test/lith_test.gleam.old +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (@hyperpolymath) -// -// Lith NIF Integration Test - -import lith -import gleam/io -import gleam/string -import gleam/option - -pub fn main() { - io.println("Testing Lith NIF integration...") - - // Test 1: Initialize Lith - case lith.init() { - Ok(_) -> io.println("✓ Lith init successful") - Error(e) -> { - io.println("✗ Lith init failed: " <> string.inspect(e)) - panic as "Lith init failed" - } - } - - // Test 2: Create a test database - case lith.create("/tmp/test_lith.db", 1000) { - Ok(db) -> { - io.println("✓ Database created successfully") - - // Test 3: Execute a SELECT query - case lith.execute_query(db, "SELECT * FROM evidence", []) { - Ok(cursor) -> { - io.println("✓ Query executed successfully") - - // Test 4: Fetch results from cursor - case lith.cursor_next(cursor) { - Ok(option.Some(json_doc)) -> { - io.println("✓ Cursor fetch successful") - io.println(" Document: " <> json_doc) - } - Ok(option.None) -> io.println("✓ No results (expected for empty database)") - Error(e) -> io.println("✗ Cursor fetch failed: " <> string.inspect(e)) - } - } - Error(e) -> io.println("✗ Query execution failed: " <> string.inspect(e)) - } - } - Error(e) -> io.println("✗ Database creation failed: " <> string.inspect(e)) - } - - io.println("\nLith NIF test complete!") -} diff --git a/lithoglyph/glyphbase/server/test_lithoglyph_nif.erl b/lithoglyph/glyphbase/server/test_lithoglyph_nif.erl deleted file mode 100755 index 3ef4cbce..00000000 --- a/lithoglyph/glyphbase/server/test_lithoglyph_nif.erl +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env escript -%% SPDX-License-Identifier: MPL-2.0 -%% Test Lith Rust NIF from FormBase - -main(_) -> - io:format("~n=== Lith NIF Test (FormBase) ===~n~n"), - - % Add paths - true = code:add_patha("ebin"), - true = code:add_patha("build/dev/erlang/formbase_server/ebin"), - - % Test 1: Version - io:format("Test 1: Calling version()...~n"), - Version = lith_nif:version(), - io:format(" ✓ Version: ~p~n~n", [Version]), - - % Test 2: Open database - io:format("Test 2: Opening database...~n"), - DbRef = lith_nif:db_open(<<"/tmp/lith_test">>), - io:format(" ✓ Database opened~n~n", []), - - % Test 3: Begin transaction - io:format("Test 3: Beginning transaction...~n"), - {ok, TxnRef} = lith_nif:txn_begin(DbRef, <<"read_write">>), - io:format(" ✓ Transaction started~n~n", []), - - % Test 4: Apply operation (with CBOR map) - io:format("Test 4: Applying operation...~n"), - CborMap = <<16#a1, 16#01, 16#02>>, % CBOR map {1: 2} - {ok, BlockId} = lith_nif:apply(TxnRef, CborMap), - io:format(" ✓ Operation applied, block ID: ~p~n~n", [BlockId]), - - % Test 5: Commit transaction - io:format("Test 5: Committing transaction...~n"), - _CommitResult = lith_nif:txn_commit(TxnRef), - io:format(" ✓ Transaction committed~n~n", []), - - % Test 6: Schema - io:format("Test 6: Getting schema...~n"), - Schema = lith_nif:schema(DbRef), - io:format(" ✓ Schema: ~p~n~n", [Schema]), - - % Test 7: Journal - io:format("Test 7: Getting journal...~n"), - Journal = lith_nif:journal(DbRef, 0), - io:format(" ✓ Journal: ~p~n~n", [Journal]), - - % Test 8: Close database - io:format("Test 8: Closing database...~n"), - _CloseResult = lith_nif:db_close(DbRef), - io:format(" ✓ Database closed~n~n", []), - - io:format("=== All tests passed! ===~n~n"), - halt(0). diff --git a/lithoglyph/glyphbase/ui/.gitkeep b/lithoglyph/glyphbase/ui/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/lithoglyph/glyphbase/ui/BUNDLE-ANALYSIS.md b/lithoglyph/glyphbase/ui/BUNDLE-ANALYSIS.md deleted file mode 100644 index 9aa4a3d3..00000000 --- a/lithoglyph/glyphbase/ui/BUNDLE-ANALYSIS.md +++ /dev/null @@ -1,235 +0,0 @@ -# Production Build Analysis - - - -**Build Date:** 2026-02-06 **Vite Version:** 6.4.1 **Build Time:** 2.02 seconds - -## Bundle Sizes - -### Raw Sizes - -- **JavaScript**: 188.13 kB (index-ZOVpEuRb.js) -- **CSS**: 28.31 kB (index-4AOLsiG8.css) -- **HTML**: 0.45 kB (index.html) -- **Total**: 216.89 kB - -### Gzipped Sizes - -- **JavaScript**: 59.76 kB (68.2% compression) -- **CSS**: 5.59 kB (80.3% compression) -- **HTML**: 0.31 kB (31.1% compression) -- **Total Gzipped**: 65.66 kB (69.7% compression) - -### Build Performance - -- Build time: 2.02 seconds -- Modules transformed: 46 -- ReScript modules: 97 - -## Size Breakdown - -| Asset | Raw | Gzipped | Compression | -| ---------- | ------------- | ------------ | ----------- | -| JavaScript | 188.13 kB | 59.76 kB | 68.2% | -| CSS | 28.31 kB | 5.59 kB | 80.3% | -| HTML | 0.45 kB | 0.31 kB | 31.1% | -| **Total** | **216.89 kB** | **65.66 kB** | **69.7%** | - -## Performance Metrics - -### ✅ Excellent Bundle Size - -- Under 200 kB raw (188 kB JS) -- Under 70 kB gzipped (60 kB JS) -- Meets performance budget for fast initial load -- **Load time estimate (3G)**: ~2-3 seconds -- **Load time estimate (4G/LTE)**: <1 second - -### ✅ Efficient Compression - -- 68% JavaScript compression ratio -- 80% CSS compression ratio -- Industry standard: 60-70% -- Brotli could further reduce by ~10-15% - -### ✅ Fast Build Time - -- 2.02 seconds total (cold build) -- Incremental builds: <1 second -- ReScript compilation: efficient type checking - -## Application Features (Included in Bundle) - -The 188 kB bundle includes: - -**Core UI:** - -- Complete spreadsheet grid with cell editing -- Drag-to-fill functionality -- Column resizing and reordering -- Row selection and bulk operations - -**Data Management:** - -- Undo/redo system -- Filter engine with multiple operators -- Sort by any column -- Hide/show columns -- Search functionality - -**Multiple Views:** - -- Grid view (default) -- Calendar view with drag-to-reschedule -- Kanban board -- Gallery view with image support -- Form view for public submissions - -**Collaboration:** - -- LiveCursors component (real-time cursor tracking) -- PresenceIndicators (online user list) -- CellComments (comment threads) -- CRDT-based state management (Yjs) - -**Data Types:** - -- Text, Number, Date, Checkbox -- Select (single/multi) -- Attachments, URLs, Emails -- Formula fields -- Rollup and Lookup fields - -**Proven Library Integration:** - -- Type-safe field validation -- Compile-time safety guarantees -- Zero-cost abstractions - -## Comparison to Similar Applications - -| Application | Bundle Size (gzipped) | Features | -| ------------------ | --------------------- | ---------------------------------------------------- | -| **Glyphbase** | **65.66 kB** | Grid, Calendar, Kanban, Gallery, Form, Collaboration | -| Airtable (minimal) | ~800 kB | Similar features | -| Notion (minimal) | ~1.2 MB | Similar features | -| Google Sheets | ~2-3 MB | Similar features | -| Baserow | ~400 kB | Similar features | - -**Result**: Glyphbase is **6-12x smaller** than comparable applications! - -## Performance Budgets - -| Metric | Budget | Actual | Status | -| ------------- | ------- | ------ | ------- | -| Initial JS | <200 kB | 188 kB | ✅ Pass | -| Initial CSS | <50 kB | 28 kB | ✅ Pass | -| Total (raw) | <300 kB | 217 kB | ✅ Pass | -| Gzipped JS | <100 kB | 60 kB | ✅ Pass | -| Gzipped Total | <150 kB | 66 kB | ✅ Pass | - -## Optimization Opportunities - -### Already Optimized ✅ - -- ReScript dead code elimination -- Vite tree-shaking -- Minification and uglification -- CSS optimization -- Efficient compression - -### Future Optimizations (Optional) - -#### 1. Code Splitting by Route - -Split views into separate chunks: - -```javascript -const CalendarView = lazy(() => import("./views/CalendarView")); -const KanbanView = lazy(() => import("./views/KanbanView")); -const GalleryView = lazy(() => import("./views/GalleryView")); -``` - -**Potential savings**: 30-50 kB per lazy-loaded view - -#### 2. Dynamic Imports for Heavy Libraries - -Load Yjs collaboration only when needed: - -```javascript -const enableCollaboration = async () => { - const Yjs = await import("yjs"); - const WebsocketProvider = await import("y-websocket"); - // Initialize collaboration -}; -``` - -**Potential savings**: ~40 kB - -#### 3. Dependency Analysis - -Check for unused dependencies: - -```bash -npm install -g depcheck -depcheck -``` - -#### 4. Brotli Compression - -Enable Brotli for additional ~10-15% compression: - -- Gzipped: 66 kB -- Brotli: ~56 kB (estimated) - -#### 5. Image Optimization - -If adding images: - -- Use WebP format -- Lazy load images -- Responsive images with srcset - -## Recommendations - -### ✅ Current Status: Production Ready - -The bundle is exceptionally lean for the features provided. No immediate -optimizations needed. - -### Optional Next Steps - -1. **Performance Monitoring** - - Add Web Vitals tracking - - Monitor real user metrics (RUM) - - Track Core Web Vitals (LCP, FID, CLS) - -2. **Progressive Enhancement** - - Add service worker for offline support - - Implement background sync - - Cache static assets - -3. **Bundle Analysis Tool** - ```bash - npm install --save-dev rollup-plugin-visualizer - ``` - Generate interactive bundle map - -4. **Lighthouse Audit** - ```bash - lighthouse https://your-domain.com --view - ``` - Target score: 95+ for Performance - -## Conclusion - -**Glyphbase achieves exceptional bundle efficiency:** - -- ✅ 65.66 kB gzipped (entire app) -- ✅ 2.02 second build time -- ✅ Production-ready performance -- ✅ 6-12x smaller than competitors -- ✅ Feature-complete and type-safe - -The ReScript + Vite stack delivers outstanding results. The application is ready -for deployment with no performance concerns. diff --git a/lithoglyph/glyphbase/ui/deno.json b/lithoglyph/glyphbase/ui/deno.json deleted file mode 100644 index ac861334..00000000 --- a/lithoglyph/glyphbase/ui/deno.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "https://deno.land/x/deno/cli/schemas/config-file.v1.json", - "compilerOptions": { - "lib": ["deno.window", "dom", "dom.iterable"], - "jsx": "react-jsx", - "jsxImportSource": "react" - }, - "imports": { - "react": "npm:react@^18.2.0", - "react-dom": "npm:react-dom@^18.2.0", - "@rescript/core": "npm:@rescript/core@^1.3.0", - "@rescript/runtime": "npm:@rescript/runtime@^12.0.2", - "jotai": "npm:jotai@^2.6.0", - "yjs": "npm:yjs@^13.6.0" - }, - "tasks": { - "setup": "echo 'Run: deno task dev'", - "dev": "deno run -A npm:vite", - "build": "deno run -A npm:vite build", - "preview": "deno run -A npm:vite preview", - "test": "deno test --allow-read --allow-env", - "lint": "deno lint", - "fmt": "deno fmt", - "storybook": "deno run -A npm:storybook dev -p 6006" - } -} diff --git a/lithoglyph/glyphbase/ui/index.html b/lithoglyph/glyphbase/ui/index.html deleted file mode 100644 index 5095beb8..00000000 --- a/lithoglyph/glyphbase/ui/index.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - Glyphbase - - - -
- - - diff --git a/lithoglyph/glyphbase/ui/package.json b/lithoglyph/glyphbase/ui/package.json deleted file mode 100644 index 27a7f3f6..00000000 --- a/lithoglyph/glyphbase/ui/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "@glyphbase/ui", - "version": "0.1.0", - "private": true, - "license": "PMPL-1.0-or-later", - "type": "module", - "scripts": { - "res:build": "rescript build", - "res:watch": "rescript build -w", - "res:clean": "rescript clean", - "dev": "vite", - "build": "rescript build && vite build", - "preview": "vite preview" - }, - "dependencies": { - "@rescript/core": "^1.6.1", - "@rescript/react": "^0.14.0", - "jotai": "^2.16.1", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "yjs": "^13.6.29" - }, - "devDependencies": { - "@vitejs/plugin-react": "^4.3.4", - "rescript": "^12.0.0", - "vite": "^6.0.0" - } -} diff --git a/lithoglyph/glyphbase/ui/rescript.json b/lithoglyph/glyphbase/ui/rescript.json deleted file mode 100644 index 5d437ec0..00000000 --- a/lithoglyph/glyphbase/ui/rescript.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "@glyphbase/ui", - "sources": [ - { - "dir": "src", - "subdirs": true - } - ], - "package-specs": [ - { - "module": "es6", - "in-source": true - } - ], - "suffix": ".res.js", - "dependencies": [ - "@rescript/core", - "@rescript/react" - ], - "jsx": { - "version": 4, - "mode": "automatic" - }, - "warnings": { - "number": "+A-42-48" - } -} diff --git a/lithoglyph/glyphbase/ui/src/App.res b/lithoglyph/glyphbase/ui/src/App.res deleted file mode 100644 index 9a63ada9..00000000 --- a/lithoglyph/glyphbase/ui/src/App.res +++ /dev/null @@ -1,1093 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -open Types - -// Type-safe DOM event listener bindings (eliminates Obj.magic) -@val external addKeydownListener: (string, Dom.keyboardEvent => unit) => unit = "document.addEventListener" -@val external removeKeydownListener: (string, Dom.keyboardEvent => unit) => unit = "document.removeEventListener" - -// Demo data for development -let demoTable: table = { - id: "tbl_demo", - name: "Projects", - primaryFieldId: "fld_name", - fields: [ - { - id: "fld_name", - name: "Name", - fieldType: Text, - required: true, - defaultValue: None, - description: None, - }, - { - id: "fld_status", - name: "Status", - fieldType: Select(["Not Started", "In Progress", "Done"]), - required: false, - defaultValue: Some("Not Started"), - description: None, - }, - { - id: "fld_tags", - name: "Tags", - fieldType: MultiSelect(["Frontend", "Backend", "Bug", "Feature", "Docs", "Urgent"]), - required: false, - defaultValue: None, - description: None, - }, - { - id: "fld_priority", - name: "Priority", - fieldType: Number, - required: false, - defaultValue: None, - description: None, - }, - { - id: "fld_due", - name: "Due Date", - fieldType: Date, - required: false, - defaultValue: None, - description: None, - }, - { - id: "fld_done", - name: "Complete", - fieldType: Checkbox, - required: false, - defaultValue: None, - description: None, - }, - ], -} - -let demoRows: array = [ - { - id: "row_1", - createdAt: "2026-01-12T00:00:00Z", - updatedAt: "2026-01-12T00:00:00Z", - cells: Dict.fromArray([ - ("fld_name", {fieldId: "fld_name", value: TextValue("Build grid component"), provenance: []}), - ("fld_status", {fieldId: "fld_status", value: SelectValue("In Progress"), provenance: []}), - ( - "fld_tags", - {fieldId: "fld_tags", value: MultiSelectValue(["Frontend", "Feature"]), provenance: []}, - ), - ("fld_priority", {fieldId: "fld_priority", value: NumberValue(1.0), provenance: []}), - ( - "fld_due", - {fieldId: "fld_due", value: DateValue(Date.fromString("2026-01-15")), provenance: []}, - ), - ("fld_done", {fieldId: "fld_done", value: CheckboxValue(false), provenance: []}), - ]), - }, - { - id: "row_2", - createdAt: "2026-01-12T00:00:00Z", - updatedAt: "2026-01-12T00:00:00Z", - cells: Dict.fromArray([ - ( - "fld_name", - {fieldId: "fld_name", value: TextValue("Implement Lith bindings"), provenance: []}, - ), - ("fld_status", {fieldId: "fld_status", value: SelectValue("Not Started"), provenance: []}), - ( - "fld_tags", - { - fieldId: "fld_tags", - value: MultiSelectValue(["Backend", "Feature", "Urgent"]), - provenance: [], - }, - ), - ("fld_priority", {fieldId: "fld_priority", value: NumberValue(2.0), provenance: []}), - ( - "fld_due", - {fieldId: "fld_due", value: DateValue(Date.fromString("2026-01-20")), provenance: []}, - ), - ("fld_done", {fieldId: "fld_done", value: CheckboxValue(false), provenance: []}), - ]), - }, - { - id: "row_3", - createdAt: "2026-01-12T00:00:00Z", - updatedAt: "2026-01-12T00:00:00Z", - cells: Dict.fromArray([ - ( - "fld_name", - {fieldId: "fld_name", value: TextValue("Add real-time collaboration"), provenance: []}, - ), - ("fld_status", {fieldId: "fld_status", value: SelectValue("Not Started"), provenance: []}), - ( - "fld_tags", - { - fieldId: "fld_tags", - value: MultiSelectValue(["Frontend", "Backend", "Feature"]), - provenance: [], - }, - ), - ("fld_priority", {fieldId: "fld_priority", value: NumberValue(3.0), provenance: []}), - ( - "fld_due", - {fieldId: "fld_due", value: DateValue(Date.fromString("2026-02-01")), provenance: []}, - ), - ("fld_done", {fieldId: "fld_done", value: CheckboxValue(false), provenance: []}), - ]), - }, -] - -// Demo presence data for collaboration features -let demoPresence: array = [ - {clientId: "user_1", name: "Alice", color: "#3b82f6", cursor: None}, - {clientId: "user_2", name: "Bob", color: "#10b981", cursor: None}, - { - clientId: "user_3", - name: "Charlie", - color: "#f59e0b", - cursor: Some({rowId: "row_1", fieldId: "fld_status"}), - }, -] - -module Sidebar = { - @react.component - let make = ( - ~bases: array, - ~currentBase: option, - ~onSelectBase: string => unit, - ~onCreateBase: unit => unit, - ~onDeleteBase: string => unit, - ~onCreateTable: unit => unit, - ~onDeleteTable: string => unit, - ) => { - - } -} - -module ViewTabs = { - @react.component - let make = () => { -
- - - - - -
- } -} - -module FilterPanel = { - @react.component - let make = ( - ~fields: array, - ~filters: array, - ~onAddFilter: GridStore.filterCondition => unit, - ~onRemoveFilter: string => unit, - ~onUpdateFilter: GridStore.filterCondition => unit, - ) => { - let operatorOptions = [ - (GridStore.Contains, "contains"), - (GridStore.DoesNotContain, "does not contain"), - (GridStore.Is, "is"), - (GridStore.IsNot, "is not"), - (GridStore.IsEmpty, "is empty"), - (GridStore.IsNotEmpty, "is not empty"), - (GridStore.GreaterThan, ">"), - (GridStore.LessThan, "<"), - (GridStore.GreaterOrEqual, ">="), - (GridStore.LessOrEqual, "<="), - ] - - let operatorToString = (op: GridStore.filterOperator): string => { - switch op { - | Contains => "contains" - | DoesNotContain => "does not contain" - | Is => "is" - | IsNot => "is not" - | IsEmpty => "is empty" - | IsNotEmpty => "is not empty" - | GreaterThan => ">" - | LessThan => "<" - | GreaterOrEqual => ">=" - | LessOrEqual => "<=" - } - } - - let stringToOperator = (s: string): GridStore.filterOperator => { - switch s { - | "contains" => Contains - | "does not contain" => DoesNotContain - | "is" => Is - | "is not" => IsNot - | "is empty" => IsEmpty - | "is not empty" => IsNotEmpty - | ">" => GreaterThan - | "<" => LessThan - | ">=" => GreaterOrEqual - | "<=" => LessOrEqual - | _ => Contains - } - } - - let handleAddFilter = () => { - let firstField = fields->Array.get(0) - switch firstField { - | Some(field) => - onAddFilter({ - id: "filter_" ++ Float.toString(Date.now()), - fieldId: field.id, - operator: Contains, - value: "", - }) - | None => () - } - } - -
-
- {React.string("Filters")} - -
-
- {filters - ->Array.map(filter => { -
- - - {switch filter.operator { - | IsEmpty | IsNotEmpty => React.null - | _ => - { - let newValue = ReactEvent.Form.target(e)["value"] - onUpdateFilter({...filter, value: newValue}) - }} - /> - }} - -
- }) - ->React.array} - {if Array.length(filters) == 0 { -
{React.string("No filters applied")}
- } else { - React.null - }} -
-
- } -} - -module HideFieldsPanel = { - @react.component - let make = ( - ~fields: array, - ~hiddenColumns: array, - ~onToggleColumn: string => unit, - ) => { -
-
- {React.string("Fields")} -
-
- {fields - ->Array.map(field => { - let isHidden = hiddenColumns->Array.includes(field.id) - - }) - ->React.array} -
-
- {React.string("Uncheck to hide fields from the grid view")} -
-
- } -} - -module Toolbar = { - @react.component - let make = ( - ~filterCount: int, - ~onToggleFilter: unit => unit, - ~showFilter: bool, - ~hiddenCount: int, - ~onToggleHideFields: unit => unit, - ~showHideFields: bool, - ~searchTerm: string, - ~onSearchChange: string => unit, - ) => { -
- - - - - { - let value = %raw(`evt.target.value`) - onSearchChange(value) - }} - /> -
- } -} - -@react.component -let make = () => { - let (rows, setRows) = React.useState(() => demoRows) - let (filters, setFilters) = Jotai.useAtom(GridStore.filtersAtom) - let (filterConjunction, _setFilterConjunction) = Jotai.useAtom(GridStore.filterConjunctionAtom) - let (sortConfig, setSortConfig) = Jotai.useAtom(GridStore.sortConfigAtom) - let (hiddenColumns, setHiddenColumns) = Jotai.useAtom(GridStore.hiddenColumnsAtom) - let (searchTerm, setSearchTerm) = Jotai.useAtom(GridStore.searchTermAtom) - let (history, setHistory) = Jotai.useAtom(GridStore.historyAtom) - let (showFilterPanel, setShowFilterPanel) = React.useState(() => false) - let (showHideFieldsPanel, setShowHideFieldsPanel) = React.useState(() => false) - - // Base and table management - let (bases, setBases) = Jotai.useAtom(BaseStore.basesAtom) - let (currentBase, setCurrentBase) = Jotai.useAtom(BaseStore.currentBaseAtom) - let (showCreateBaseModal, setShowCreateBaseModal) = React.useState(() => false) - let (showCreateTableModal, setShowCreateTableModal) = React.useState(() => false) - let (showDeleteBaseModal, setShowDeleteBaseModal) = React.useState(() => false) - let (showDeleteTableModal, setShowDeleteTableModal) = React.useState(() => false) - let (baseToDelete, setBaseToDelete) = React.useState(() => None) - let (tableToDelete, setTableToDelete) = React.useState(() => None) - let (newBaseName, setNewBaseName) = React.useState(() => "") - let (newTableName, setNewTableName) = React.useState(() => "") - - // Initialize with demo base if no bases exist - React.useEffect1(() => { - if Array.length(bases) == 0 { - let demoBase = BaseStore.createBase("Demo Base", Some("📊")) - let demoTableWithBase = BaseStore.addTableToBase(demoBase, demoTable) - setBases(_ => [demoTableWithBase]) - setCurrentBase(_ => Some(demoTableWithBase)) - } - None - }, []) - - // Apply search, filters, and sorting to get visible rows - let searchedRows = GridStore.applySearch(rows, searchTerm) - let filteredRows = GridStore.applyFilters(searchedRows, filters, filterConjunction) - let sortedRows = GridStore.applySort(filteredRows, sortConfig) - - // Filter handlers - let handleAddFilter = (filter: GridStore.filterCondition) => { - setFilters(prev => Array.concat(prev, [filter])) - } - - let handleRemoveFilter = (filterId: string) => { - setFilters(prev => prev->Array.filter(f => f.id != filterId)) - } - - let handleUpdateFilter = (filter: GridStore.filterCondition) => { - setFilters(prev => prev->Array.map(f => f.id == filter.id ? filter : f)) - } - - let handleToggleFilterPanel = () => { - setShowFilterPanel(prev => !prev) - setShowHideFieldsPanel(_ => false) - } - - let handleToggleHideFieldsPanel = () => { - setShowHideFieldsPanel(prev => !prev) - setShowFilterPanel(_ => false) - } - - // Toggle column visibility - let handleToggleColumn = (fieldId: string) => { - setHiddenColumns(prev => { - if prev->Array.includes(fieldId) { - prev->Array.filter(id => id != fieldId) - } else { - Array.concat(prev, [fieldId]) - } - }) - } - - // Sort handler - toggle direction or set new field - let handleSort = (fieldId: string) => { - setSortConfig(prev => { - switch prev { - | Some({fieldId: currentFieldId, direction}) if currentFieldId == fieldId => - // Same field - toggle direction or clear - switch direction { - | #Asc => Some({GridStore.fieldId, direction: #Desc}) - | #Desc => None // Clear sort on third click - } - | _ => - // Different field - sort ascending - Some({GridStore.fieldId, direction: #Asc}) - } - }) - } - - // Base management handlers - let handleSelectBase = (baseId: string) => { - let selected = bases->Array.find(b => b.id == baseId) - setCurrentBase(_ => selected) - } - - let handleCreateBase = () => { - if newBaseName->String.trim != "" { - let newBase = BaseStore.createBase(newBaseName, None) - setBases(prev => Array.concat(prev, [newBase])) - setNewBaseName(_ => "") - setShowCreateBaseModal(_ => false) - setCurrentBase(_ => Some(newBase)) - } - } - - let handleDeleteBase = (baseId: string) => { - setBaseToDelete(_ => Some(baseId)) - setShowDeleteBaseModal(_ => true) - } - - let confirmDeleteBase = () => { - switch baseToDelete { - | Some(baseId) => { - setBases(prev => prev->Array.filter(b => b.id != baseId)) - // If deleting current base, clear selection - switch currentBase { - | Some(cb) if cb.id == baseId => setCurrentBase(_ => None) - | _ => () - } - setShowDeleteBaseModal(_ => false) - setBaseToDelete(_ => None) - } - | None => () - } - } - - // Table management handlers - let handleCreateTable = () => { - switch currentBase { - | Some(base) => - if newTableName->String.trim != "" { - let newTable = BaseStore.createTable(base.id, newTableName, "fld_name") - let updatedBase = BaseStore.addTableToBase(base, newTable) - setBases(prev => prev->Array.map(b => b.id == base.id ? updatedBase : b)) - setCurrentBase(_ => Some(updatedBase)) - setNewTableName(_ => "") - setShowCreateTableModal(_ => false) - } - | None => Console.log("No base selected") - } - } - - let handleDeleteTable = (tableId: string) => { - setTableToDelete(_ => Some(tableId)) - setShowDeleteTableModal(_ => true) - } - - let confirmDeleteTable = () => { - switch (currentBase, tableToDelete) { - | (Some(base), Some(tableId)) => { - let updatedBase = BaseStore.removeTableFromBase(base, tableId) - setBases(prev => prev->Array.map(b => b.id == base.id ? updatedBase : b)) - setCurrentBase(_ => Some(updatedBase)) - setShowDeleteTableModal(_ => false) - setTableToDelete(_ => None) - } - | _ => () - } - } - - // Convert cellValue to JSON for API - let cellValueToJson = (value: cellValue): JSON.t => { - switch value { - | TextValue(s) => JSON.Encode.string(s) - | NumberValue(n) => JSON.Encode.float(n) - | CheckboxValue(b) => JSON.Encode.bool(b) - | SelectValue(s) => JSON.Encode.string(s) - | MultiSelectValue(arr) => JSON.Encode.array(arr->Array.map(JSON.Encode.string)) - | DateValue(d) => JSON.Encode.string(Date.toISOString(d)) - | LinkValue(ids) => JSON.Encode.array(ids->Array.map(JSON.Encode.string)) - | NullValue => JSON.Encode.null - | _ => JSON.Encode.null - } - } - - // Handle cell updates - let handleCellUpdate = (rowId: string, fieldId: string, newValue: cellValue) => { - // Get old value for undo/redo - let oldValue = - rows - ->Array.find(r => r.id == rowId) - ->Option.flatMap(r => r.cells->Dict.get(fieldId)) - ->Option.map(c => c.value) - ->Option.getOr(NullValue) - - // Record to history - setHistory(prev => GridStore.recordEdit(prev, rowId, fieldId, oldValue, newValue)) - - // Optimistic update - update local state first - setRows(prevRows => { - prevRows->Array.map(row => { - if row.id == rowId { - let newCells = - row.cells - ->Dict.toArray - ->Array.map( - ((key, cell)) => { - if key == fieldId { - ( - key, - { - ...cell, - value: newValue, - }, - ) - } else { - (key, cell) - } - }, - ) - ->Dict.fromArray - - // Add the cell if it doesn't exist - if !(newCells->Dict.keysToArray->Array.includes(fieldId)) { - newCells->Dict.set( - fieldId, - { - fieldId, - value: newValue, - provenance: [], - }, - ) - } - - { - ...row, - cells: newCells, - updatedAt: Date.toISOString(Date.make()), - } - } else { - row - } - }) - }) - - // Sync with backend via API - let _ = Client.updateCell( - "base_demo", - demoTable.id, - rowId, - fieldId, - cellValueToJson(newValue), - (), - )->Promise.thenResolve(result => { - switch result { - | Ok(_) => Console.log("Cell updated successfully") - | Error(err) => Console.error2("Failed to update cell:", err.message) - } - }) - } - - // Handle undo - let handleUndo = () => { - if GridStore.canUndo(history) { - let (newHistory, action) = GridStore.performUndo(history) - setHistory(_ => newHistory) - - switch action { - | Some({rowId, fieldId, oldValue, newValue: _}) => { - // Apply the old value - setRows(prevRows => { - prevRows->Array.map(row => { - if row.id == rowId { - let newCells = - row.cells - ->Dict.toArray - ->Array.map( - ((key, cell)) => { - if key == fieldId { - (key, {...cell, value: oldValue}) - } else { - (key, cell) - } - }, - ) - ->Dict.fromArray - - {...row, cells: newCells, updatedAt: Date.toISOString(Date.make())} - } else { - row - } - }) - }) - - // Sync with backend - let _ = Client.updateCell( - "base_demo", - demoTable.id, - rowId, - fieldId, - cellValueToJson(oldValue), - (), - ) - } - | None => () - } - } - } - - // Handle redo - let handleRedo = () => { - if GridStore.canRedo(history) { - let (newHistory, action) = GridStore.performRedo(history) - setHistory(_ => newHistory) - - switch action { - | Some({rowId, fieldId, oldValue: _, newValue}) => { - // Apply the new value - setRows(prevRows => { - prevRows->Array.map(row => { - if row.id == rowId { - let newCells = - row.cells - ->Dict.toArray - ->Array.map( - ((key, cell)) => { - if key == fieldId { - (key, {...cell, value: newValue}) - } else { - (key, cell) - } - }, - ) - ->Dict.fromArray - - {...row, cells: newCells, updatedAt: Date.toISOString(Date.make())} - } else { - row - } - }) - }) - - // Sync with backend - let _ = Client.updateCell( - "base_demo", - demoTable.id, - rowId, - fieldId, - cellValueToJson(newValue), - (), - ) - } - | None => () - } - } - } - - // Handle adding a new row - let handleAddRow = () => { - let newRowId = - "row_" ++ Int.toString(Array.length(rows) + 1) ++ "_" ++ Float.toString(Date.now()) - let now = Date.toISOString(Date.make()) - - // Create empty cells with default values - let emptyCells = - demoTable.fields - ->Array.map(field => { - let defaultValue = switch field.defaultValue { - | Some(v) => TextValue(v) - | None => NullValue - } - (field.id, {fieldId: field.id, value: defaultValue, provenance: []}) - }) - ->Dict.fromArray - - let newRow: row = { - id: newRowId, - cells: emptyCells, - createdAt: now, - updatedAt: now, - } - - // Optimistic update - add to local state first - setRows(prevRows => Array.concat(prevRows, [newRow])) - - // Sync with backend via API - let cellsJson = - emptyCells - ->Dict.toArray - ->Array.map(((fieldId, cell)) => { - (fieldId, cellValueToJson(cell.value)) - }) - ->Dict.fromArray - - let _ = Client.createRow( - "base_demo", - demoTable.id, - newRowId, - cellsJson, - )->Promise.thenResolve(result => { - switch result { - | Ok(_) => Console.log2("Row created successfully:", newRowId) - | Error(err) => Console.error2("Failed to create row:", err.message) - } - }) - } - - // Handle deleting a row - let handleDeleteRow = (rowId: string) => { - // Optimistic update - remove from local state first - setRows(prevRows => prevRows->Array.filter(row => row.id != rowId)) - - // Sync with backend via API - let _ = Client.deleteRow("base_demo", demoTable.id, rowId)->Promise.thenResolve(result => { - switch result { - | Ok(_) => Console.log2("Row deleted successfully:", rowId) - | Error(err) => Console.error2("Failed to delete row:", err.message) - } - }) - } - - // Keyboard shortcuts for undo/redo - React.useEffect0(() => { - let handleKeyDown = (evt: Dom.keyboardEvent) => { - let ctrlOrCmd = %raw(`evt.ctrlKey || evt.metaKey`) - let shift = %raw(`evt.shiftKey`) - let key = %raw(`evt.key`) - - // Ctrl+Z or Cmd+Z for undo - if ctrlOrCmd && !shift && key == "z" { - %raw(`evt.preventDefault()`) - handleUndo() - } - - // Ctrl+Y or Ctrl+Shift+Z or Cmd+Shift+Z for redo - if (ctrlOrCmd && key == "y") || (ctrlOrCmd && shift && key == "z") { - %raw(`evt.preventDefault()`) - handleRedo() - } - } - - // Type-safe keydown listener via external binding (no Obj.magic) - addKeydownListener("keydown", handleKeyDown) - - Some( - () => { - removeKeydownListener("keydown", handleKeyDown) - }, - ) - }) - - -
-
-

{React.string("FormBase")}

-

{React.string("Open-source Airtable alternative with provenance tracking")}

-
-
- setShowCreateBaseModal(_ => true)} - onDeleteBase={handleDeleteBase} - onCreateTable={() => setShowCreateTableModal(_ => true)} - onDeleteTable={handleDeleteTable} - /> -
- - setSearchTerm(_ => value)} - /> - {if showHideFieldsPanel { - - } else if showFilterPanel { - - } else { - React.null - }} -
- - -
- -
-
- - setShowCreateBaseModal(_ => false)} - title="Create New Base" - > -
-
- - { - let value = %raw(`evt.target.value`) - setNewBaseName(_ => value) - }} - onKeyDown={evt => { - if evt->ReactEvent.Keyboard.key == "Enter" { - handleCreateBase() - } - }} - /> -
-
- - -
-
- - - setShowCreateTableModal(_ => false)} - title="Create New Table" - > -
-
- - { - let value = %raw(`evt.target.value`) - setNewTableName(_ => value) - }} - onKeyDown={evt => { - if evt->ReactEvent.Keyboard.key == "Enter" { - handleCreateTable() - } - }} - /> -
-
- - -
-
- - - setShowDeleteBaseModal(_ => false)} - title="Delete Base" - > -
-

{React.string("Are you sure you want to delete this base and all its tables?")}

-
- - -
-
- - - setShowDeleteTableModal(_ => false)} - title="Delete Table" - > -
-

{React.string("Are you sure you want to delete this table?")}

-
- - -
-
- -
- -} diff --git a/lithoglyph/glyphbase/ui/src/Main.res b/lithoglyph/glyphbase/ui/src/Main.res deleted file mode 100644 index 563a4e68..00000000 --- a/lithoglyph/glyphbase/ui/src/Main.res +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Main entry point with formally verified DOM mounting - -// Use SafeDOM for high-assurance mount operations -// Provides compile-time guarantees: no null pointers, validated selectors, type-safe operations - -module SafeDOMMounter = { - // Safe React root mounting with error handling - let mountReactRoot = (selector: string, ~onError: string => unit): unit => { - SafeDOM.mountWhenReady( - selector, - "", // Empty HTML - we'll use ReactDOM to render - ~onSuccess=element => { - // Element proven to exist - mount React root - let root = ReactDOM.Client.createRoot(element) - ReactDOM.Client.Root.render(root, ) - }, - ~onError=err => { - onError(err) - }, - ) - } -} - -// Mount with formally verified selector -SafeDOMMounter.mountReactRoot("#root", ~onError=err => { - Console.error("Failed to mount Glyphbase:") - Console.error(err) -}) diff --git a/lithoglyph/glyphbase/ui/src/Types.res b/lithoglyph/glyphbase/ui/src/Types.res deleted file mode 100644 index 8628a262..00000000 --- a/lithoglyph/glyphbase/ui/src/Types.res +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Core types for Glyphbase - -type fieldType = - | Text - | Number - | Select(array) - | MultiSelect(array) - | Date - | DateTime - | Checkbox - | Link(string) // table id - | Attachment - | Formula(string) - | Rollup(string, string) // linked field, aggregation - | Lookup(string, string) // linked field, lookup field - | Url - | Email - | Phone - | Rating - | Barcode - -type fieldConfig = { - id: string, - name: string, - fieldType: fieldType, - required: bool, - defaultValue: option, - description: option, -} - -// Attachment file object -type attachmentFile = { - id: string, - name: string, - url: string, - mimeType: string, - size: int, -} - -type cellValue = - | TextValue(string) - | NumberValue(float) - | SelectValue(string) - | MultiSelectValue(array) - | DateValue(Date.t) - | CheckboxValue(bool) - | LinkValue(array) // row ids - | AttachmentValue(array) - | UrlValue(string) // URL field type - | EmailValue(string) // Email field type - | PhoneValue(string) // Phone field type - | NullValue - -type provenanceEntry = { - timestamp: string, - userId: string, - userName: string, - previousValue: option, - newValue: cellValue, - rationale: option, -} - -type cell = { - fieldId: string, - value: cellValue, - provenance: array, -} - -type row = { - id: string, - cells: dict, - createdAt: string, - updatedAt: string, -} - -type table = { - id: string, - name: string, - fields: array, - primaryFieldId: string, -} - -type viewType = - | Grid - | Kanban(string) // group by field id - | Calendar(string) // date field id - | Gallery(string) // image field id - | Form - -type viewConfig = { - id: string, - name: string, - tableId: string, - viewType: viewType, - visibleFields: array, - sortBy: option<(string, [#Asc | #Desc])>, - filterBy: option, // GQL filter expression -} - -type base = { - id: string, - name: string, - description: option, - icon: option, - tables: array, - views: array, - createdAt: string, - updatedAt: string, -} diff --git a/lithoglyph/glyphbase/ui/src/api/Client.res b/lithoglyph/glyphbase/ui/src/api/Client.res deleted file mode 100644 index 41d1e2c1..00000000 --- a/lithoglyph/glyphbase/ui/src/api/Client.res +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// API client for Glyphbase server - -let baseUrl = "http://localhost:8080/api" - -type apiError = { - code: string, - message: string, -} - -type apiResult<'a> = result<'a, apiError> - -// Generic fetch wrapper -let fetchJson = async (~method: string, ~path: string, ~body: option=?, ()): result< - JSON.t, - apiError, -> => { - let headers = Dict.fromArray([ - ("Content-Type", "application/json"), - ("Accept", "application/json"), - ]) - - let init = { - "method": method, - "headers": headers, - "body": body->Option.map(j => JSON.stringify(j))->Nullable.fromOption, - } - - try { - let response = await Fetch.fetch(baseUrl ++ path, init) - let json = await Fetch.Response.json(response) - - if Fetch.Response.ok(response) { - Ok(json) - } else { - Error({ - code: "API_ERROR", - message: "Request failed", - }) - } - } catch { - | _ => - Error({ - code: "NETWORK_ERROR", - message: "Failed to connect to server", - }) - } -} - -// Base CRUD -let getBases = async () => { - await fetchJson(~method="GET", ~path="/bases", ()) -} - -let getBase = async (id: string) => { - await fetchJson(~method="GET", ~path="/bases/" ++ id, ()) -} - -let createBase = async (name: string, description: option) => { - let body = Dict.fromArray([ - ("name", JSON.Encode.string(name)), - ("description", description->Option.mapOr(JSON.Encode.null, JSON.Encode.string)), - ]) - await fetchJson(~method="POST", ~path="/bases", ~body=JSON.Encode.object(body), ()) -} - -// Table CRUD -let getTables = async (baseId: string) => { - await fetchJson(~method="GET", ~path="/bases/" ++ baseId ++ "/tables", ()) -} - -let createTable = async (baseId: string, name: string) => { - let body = Dict.fromArray([("name", JSON.Encode.string(name))]) - await fetchJson( - ~method="POST", - ~path="/bases/" ++ baseId ++ "/tables", - ~body=JSON.Encode.object(body), - (), - ) -} - -// Row CRUD -@val external encodeURIComponent: string => string = "encodeURIComponent" - -let getRows = async (baseId: string, tableId: string, ~filter: option=?, ()) => { - let path = "/bases/" ++ baseId ++ "/tables/" ++ tableId ++ "/rows" - let queryPath = switch filter { - | Some(f) => path ++ "?filter=" ++ encodeURIComponent(f) - | None => path - } - await fetchJson(~method="GET", ~path=queryPath, ()) -} - -let createRow = async (baseId: string, tableId: string, rowId: string, cells: Dict.t) => { - let body = Dict.fromArray([ - ("id", JSON.Encode.string(rowId)), - ("cells", JSON.Encode.object(cells)), - ]) - await fetchJson( - ~method="POST", - ~path="/bases/" ++ baseId ++ "/tables/" ++ tableId ++ "/rows", - ~body=JSON.Encode.object(body), - (), - ) -} - -let updateCell = async ( - baseId: string, - tableId: string, - rowId: string, - fieldId: string, - value: JSON.t, - ~rationale: option=?, - (), -) => { - let body = Dict.fromArray([ - ("value", value), - ("rationale", rationale->Option.mapOr(JSON.Encode.null, JSON.Encode.string)), - ]) - await fetchJson( - ~method="PATCH", - ~path="/bases/" ++ baseId ++ "/tables/" ++ tableId ++ "/rows/" ++ rowId ++ "/cells/" ++ fieldId, - ~body=JSON.Encode.object(body), - (), - ) -} - -let deleteRow = async (baseId: string, tableId: string, rowId: string) => { - await fetchJson( - ~method="DELETE", - ~path="/bases/" ++ baseId ++ "/tables/" ++ tableId ++ "/rows/" ++ rowId, - (), - ) -} - -// Provenance -let getCellProvenance = async (baseId: string, tableId: string, rowId: string, fieldId: string) => { - await fetchJson( - ~method="GET", - ~path="/bases/" ++ - baseId ++ - "/tables/" ++ - tableId ++ - "/rows/" ++ - rowId ++ - "/cells/" ++ - fieldId ++ "/provenance", - (), - ) -} diff --git a/lithoglyph/glyphbase/ui/src/bindings/Fetch.res b/lithoglyph/glyphbase/ui/src/bindings/Fetch.res deleted file mode 100644 index c6f2ca3f..00000000 --- a/lithoglyph/glyphbase/ui/src/bindings/Fetch.res +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Fetch API bindings for ReScript - -type response - -@val external fetch: (string, 'options) => promise = "fetch" - -module Response = { - @get external ok: response => bool = "ok" - @get external status: response => int = "status" - @get external statusText: response => string = "statusText" - @send external json: response => promise = "json" - @send external text: response => promise = "text" -} diff --git a/lithoglyph/glyphbase/ui/src/bindings/Jotai.res b/lithoglyph/glyphbase/ui/src/bindings/Jotai.res deleted file mode 100644 index c4cf133a..00000000 --- a/lithoglyph/glyphbase/ui/src/bindings/Jotai.res +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// ReScript bindings for Jotai - -type atom<'a> -type writableAtom<'a, 'b> - -@module("jotai") -external atom: 'a => atom<'a> = "atom" - -@module("jotai") -external atomWithDefault: (unit => 'a) => atom<'a> = "atom" - -// For derived atoms, use %raw to avoid type constraints -// The get function can retrieve any atom type -@module("jotai") -external derivedAtomRaw: ('a => 'b) => atom<'b> = "atom" - -@module("jotai") -external useAtom: atom<'a> => ('a, ('a => 'a) => unit) = "useAtom" - -@module("jotai") -external useAtomValue: atom<'a> => 'a = "useAtomValue" - -@module("jotai") -external useSetAtom: atom<'a> => ('a => 'a) => unit = "useSetAtom" - -module Provider = { - @module("jotai") @react.component - external make: (~children: React.element) => React.element = "Provider" -} diff --git a/lithoglyph/glyphbase/ui/src/bindings/Webapi.res b/lithoglyph/glyphbase/ui/src/bindings/Webapi.res deleted file mode 100644 index be562b50..00000000 --- a/lithoglyph/glyphbase/ui/src/bindings/Webapi.res +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Webapi bindings for DOM operations - -module Dom = { - @val external document: Dom.document = "document" - - module Document = { - @send - external addEventListener: (Dom.document, string, 'a => unit) => unit = "addEventListener" - - @send - external removeEventListener: (Dom.document, string, 'a => unit) => unit = "removeEventListener" - - let addMouseMoveEventListener = (doc: Dom.document, handler: Dom.mouseEvent => unit) => { - addEventListener(doc, "mousemove", handler) - } - - let removeMouseMoveEventListener = (doc: Dom.document, handler: Dom.mouseEvent => unit) => { - removeEventListener(doc, "mousemove", handler) - } - - let addMouseUpEventListener = (doc: Dom.document, handler: Dom.mouseEvent => unit) => { - addEventListener(doc, "mouseup", handler) - } - - let removeMouseUpEventListener = (doc: Dom.document, handler: Dom.mouseEvent => unit) => { - removeEventListener(doc, "mouseup", handler) - } - } -} diff --git a/lithoglyph/glyphbase/ui/src/bindings/Yjs.res b/lithoglyph/glyphbase/ui/src/bindings/Yjs.res deleted file mode 100644 index 8d8ee9ae..00000000 --- a/lithoglyph/glyphbase/ui/src/bindings/Yjs.res +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// ReScript bindings for Yjs - CRDT library for real-time collaboration - -// Y.Doc - The main Yjs document -type doc - -@module("yjs") @new -external createDoc: unit => doc = "Doc" - -// Y.Map - Collaborative map structure -type map<'a> - -@send -external getMap: (doc, string) => map<'a> = "getMap" - -@send -external mapSet: (map<'a>, string, 'a) => unit = "set" - -@send -external mapGet: (map<'a>, string) => option<'a> = "get" - -@send -external mapDelete: (map<'a>, string) => unit = "delete" - -@send -external mapHas: (map<'a>, string) => bool = "has" - -@send -external mapSize: map<'a> => int = "size" - -@send -external mapClear: map<'a> => unit = "clear" - -@send -external mapForEach: (map<'a>, ('a, string) => unit) => unit = "forEach" - -// Y.Array - Collaborative array structure -type array<'a> - -@send -external getArray: (doc, string) => array<'a> = "getArray" - -@send -external arrayPush: (array<'a>, array<'a>) => unit = "push" - -@send -external arrayDelete: (array<'a>, int, int) => unit = "delete" - -@send -external arrayGet: (array<'a>, int) => option<'a> = "get" - -@send -external arrayLength: array<'a> => int = "length" - -@send -external arrayToArray: array<'a> => array<'a> = "toArray" - -// Y.Text - Collaborative text structure -type text - -@send -external getText: (doc, string) => text = "getText" - -@send -external textInsert: (text, int, string) => unit = "insert" - -@send -external textDelete: (text, int, int) => unit = "delete" - -@send -external textToString: text => string = "toString" - -@send -external textLength: text => int = "length" - -// Transactions -type transaction - -@send -external transact: (doc, transaction => unit) => unit = "transact" - -// Event system -type event<'a> - -type observeCallback<'a> = (event<'a>, transaction) => unit - -@send -external observe: (map<'a>, observeCallback<'a>) => unit = "observe" - -@send -external observeDeep: (doc, (array>, transaction) => unit) => unit = "observeDeep" - -@send -external unobserve: (map<'a>, observeCallback<'a>) => unit = "unobserve" - -// Awareness protocol (inline implementation for y-websocket compatibility) -type awareness - -@module("yjs") @new -external createAwarenessFromDoc: doc => awareness = "Awareness" - -@send -external setLocalState: (awareness, 'a) => unit = "setLocalState" - -@send -external getLocalState: awareness => option<'a> = "getLocalState" - -@send -external getStates: awareness => dict<'a> = "getStates" - -@send -external onAwarenessChange: (awareness, {..} => unit) => unit = "on" - -// WebSocket Provider (simplified inline version) -type websocketProvider = { - doc: doc, - url: string, - roomname: string, - awareness: awareness, - mutable synced: bool, - mutable connected: bool, -} - -// Create provider (note: actual WebSocket connection would need y-websocket package) -let createWebSocketProvider = (url: string, roomname: string, doc: doc): websocketProvider => { - { - doc, - url, - roomname, - awareness: createAwarenessFromDoc(doc), - synced: false, - connected: false, - } -} - -let providerConnect = (_provider: websocketProvider): unit => { - Console.log("WebSocket provider connect (stub)") -} - -let providerDisconnect = (_provider: websocketProvider): unit => { - Console.log("WebSocket provider disconnect (stub)") -} - -let providerDestroy = (_provider: websocketProvider): unit => { - Console.log("WebSocket provider destroy (stub)") -} - -let providerOn = (_provider: websocketProvider, _event: string, _callback: unit => unit): unit => { - Console.log("WebSocket provider on (stub)") -} - -let providerAwareness = (provider: websocketProvider): awareness => provider.awareness - -let providerSynced = (provider: websocketProvider): bool => provider.synced diff --git a/lithoglyph/glyphbase/ui/src/components/CellComments.res b/lithoglyph/glyphbase/ui/src/components/CellComments.res deleted file mode 100644 index 9a0db898..00000000 --- a/lithoglyph/glyphbase/ui/src/components/CellComments.res +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Cell comments component with @mention support - -type comment = { - id: string, - author: string, - authorId: string, - content: string, - mentions: array, - createdAt: Date.t, - updatedAt: Date.t, -} - -@react.component -let make = ( - ~rowId: string, - ~fieldId: string, - ~comments: array, - ~onAddComment: string => unit, - ~onClose: unit => unit, -) => { - let (newComment, setNewComment) = React.useState(() => "") - let (showMentionMenu, setShowMentionMenu) = React.useState(() => false) - let (mentionQuery, setMentionQuery) = React.useState(() => "") - - // Parse @mentions from comment text - let parseMentions = (text: string): array => { - let mentionRegex = /@(\w+)/g - // Extract all @mentions from text - [] // Placeholder - would need proper regex matching - } - - let handleSubmitComment = () => { - if newComment->String.trim != "" { - onAddComment(newComment) - setNewComment(_ => "") - } - } - - let handleInputChange = (value: string) => { - setNewComment(_ => value) - - // Detect @mention trigger - let lastAtIndex = value->String.lastIndexOf("@") - if lastAtIndex >= 0 { - let afterAt = value->String.slice(~start=lastAtIndex + 1) - if afterAt->String.includes(" ") { - setShowMentionMenu(_ => false) - } else { - setShowMentionMenu(_ => true) - setMentionQuery(_ => afterAt) - } - } else { - setShowMentionMenu(_ => false) - } - } - -
-
-

{React.string("Comments")}

- -
-
- {if Array.length(comments) == 0 { -
- {React.string("No comments yet. Be the first to comment!")} -
- } else { - comments - ->Array.map(comment => { -
-
- {React.string(comment.author)} - - {React.string(Date.toISOString(comment.createdAt)->String.slice(~start=0, ~end=16))} - -
-
{React.string(comment.content)}
- {if Array.length(comment.mentions) > 0 { -
- {comment.mentions - ->Array.map(mention => { - {React.string(`@${mention}`)} - }) - ->React.array} -
- } else { - React.null - }} -
- }) - ->React.array - }} -
-
-