diff --git a/Cargo.toml b/Cargo.toml index 5709509..a47b0d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,3 +186,7 @@ tinyagents = { path = "vendor/tinyagents" } lto = "thin" codegen-units = 1 strip = "debuginfo" + +[[example]] +name = "tinycortex" +required-features = ["tinycortex"] diff --git a/README.md b/README.md index 941c18b..d543f32 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,79 @@ workspace builds without it — `core` names `tinyagents` and `tinycortex` by path through `vendor/`, so an uninitialized checkout fails at manifest resolution rather than at compile time, which reads as a confusing error. +## Using from your project + +None of these crates are on crates.io yet, so you take the facade by git. +Which patch table you need depends on the engine you pick. + +**Remote engines (Supermemory, Mem0, Cognee — hosted or self-hosted) — no patch table:** + +```toml +[dependencies] +tinymemory = { git = "https://github.com/tinyhumansai/tinymemory", features = ["supermemory"] } +``` + +```rust,ignore +use std::sync::Arc; + +let backend = tinymemory::remote::SupermemoryMemory::cloud("sm_...")?; +let provider = Arc::new(tinymemory::remote::supermemory_provider(backend)); +``` + +The remote adapter reaches only crates.io dependencies, so cargo resolves it +without any `[patch]` entries. + +**The embedded engine (TinyCortex) — three patch entries:** + +```toml +[dependencies] +tinymemory = { git = "https://github.com/tinyhumansai/tinymemory", features = ["tinycortex"] } + +# The engine and its api are unpublished; without these, cargo resolves a +# second copy of each from the network and type identities split at the seam. +[patch.crates-io] +tinycortex = { git = "https://github.com/tinyhumansai/tinycortex" } +tinycortex-api = { git = "https://github.com/tinyhumansai/tinycortex" } +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { git = "https://github.com/tinyhumansai/tinymemory" } +``` + +```rust,ignore +use std::sync::Arc; +use tinymemory::tinycortex::{provider, InMemoryMemoryStore}; + +let provider = Arc::new(provider(Arc::new(InMemoryMemoryStore::new()))); +``` + +That is a complete embedded setup for the mandatory three families. The full +eighteen-family engine (`TinycortexProvider`) additionally needs the host +seams (`EmbeddingHost` et al.) installed — see +`adapters/tinycortex/tests/full_provider_conformance.rs` for the minimal +working wiring. + +| Feature | Engine | Class | Families served | +| --- | --- | --- | --- | +| `tinycortex` | TinyCortex, in-process | embedded | 3 (mandatory) via `provider`; all 18 via `TinycortexProvider` | +| `supermemory` | Supermemory, hosted | external | 3 (mandatory) | +| `mem0` | Mem0, self-hosted | external | 3 (mandatory) | +| `cognee` | Cognee, hosted or self-hosted | external | 3 (mandatory) | +| `memory-git` | add-on: git-backed diff snapshots | — | requires `tinycortex` | +| *(none)* | `NullMemoryProvider` | null | contract + registry only, 40 crates | + +The `namespace` driver id you may see in the registry's reserved table is +host-internal: it names `tinymemory-core`'s own store, whose constructors live +in that crate — it is not selectable from the facade. + +**A note on remote-engine performance:** recall is native to each hosted API, +but exact-CRUD operations (`get`, `list`, `count`, upsert-by-key) are +enumeration-based — the adapter pages the hosted API to find the record. Fine +for assistant-memory workloads; wrong for high-volume keyed storage. + ## The contract `MemoryProvider` is an object-safe trait with **three mandatory** capability -families and **ten optional** ones. The mandatory three are supertraits, so a -driver missing any of them cannot be constructed; the optional ten are reached +families and **fifteen optional** ones. The mandatory three are supertraits, so a +driver missing any of them cannot be constructed; the optional fifteen are reached through `as_ingest()` / `as_tree()` / … accessors that default to `None`, so a minimal driver implements what it supports and inherits correct absence for everything else. diff --git a/adapters/remote/src/common.rs b/adapters/remote/src/common.rs index 341ce13..96b42d7 100644 --- a/adapters/remote/src/common.rs +++ b/adapters/remote/src/common.rs @@ -91,6 +91,31 @@ impl HttpClient { } /// Sends a JSON request and decodes a successful JSON response. + /// The error for a non-success status, written for the operator reading a + /// log: it names the endpoint host (never the credential) and calls out a + /// rejected credential specifically, because "HTTP 401" three layers deep + /// in an anyhow chain reads as "the engine is down" and sends the operator + /// to the wrong runbook. + fn status_error(&self, path: &str, status: reqwest::StatusCode) -> anyhow::Error { + let host = self.endpoint.host_str().unwrap_or(""); + match status.as_u16() { + 401 | 403 => { + let hint = match &self.auth { + Auth::ApiKey(_) => "check the API key", + Auth::Bearer(_) => "check the bearer token", + Auth::None => { + "the endpoint requires credentials this client was not configured with" + } + }; + anyhow::anyhow!( + "memory API {path} on {host}: the configured credential was rejected \ + (HTTP {status}) — {hint}" + ) + } + _ => anyhow::anyhow!("memory API {path} on {host} returned HTTP {status}"), + } + } + pub(crate) async fn json( &self, method: Method, @@ -101,10 +126,14 @@ impl HttpClient { if let Some(body) = body { request = request.json(body); } - let response = request.send().await.context("memory API request failed")?; + let host = self.endpoint.host_str().unwrap_or("").to_owned(); + let response = request + .send() + .await + .with_context(|| format!("memory API request to {host} failed"))?; let status = response.status(); if !status.is_success() { - bail!("memory API {path} returned HTTP {status}"); + return Err(self.status_error(path, status)); } response .json() @@ -114,10 +143,15 @@ impl HttpClient { /// Sends a request and returns a successful response body as text. pub(crate) async fn text(&self, method: Method, path: &str) -> anyhow::Result { - let response = self.request(method, path)?.send().await?; + let host = self.endpoint.host_str().unwrap_or("").to_owned(); + let response = self + .request(method, path)? + .send() + .await + .with_context(|| format!("memory API request to {host} failed"))?; let status = response.status(); if !status.is_success() { - bail!("memory API {path} returned HTTP {status}"); + return Err(self.status_error(path, status)); } response .text() @@ -136,10 +170,14 @@ impl HttpClient { if let Some(body) = body { request = request.json(body); } - let response = request.send().await?; + let host = self.endpoint.host_str().unwrap_or("").to_owned(); + let response = request + .send() + .await + .with_context(|| format!("memory API request to {host} failed"))?; let status = response.status(); if !status.is_success() { - bail!("memory API {path} returned HTTP {status}"); + return Err(self.status_error(path, status)); } Ok(status) } diff --git a/adapters/remote/src/conformance_test.rs b/adapters/remote/src/conformance_test.rs index 3694b63..8adca44 100644 --- a/adapters/remote/src/conformance_test.rs +++ b/adapters/remote/src/conformance_test.rs @@ -37,6 +37,13 @@ struct Row { id: String, content: String, metadata: Value, + /// The `containerTag` the adapter sent at create time. The real service + /// files the row under exactly this tag and answers tag-filtered lists + /// with it; the double must do the same, or a lookup scoped to the tag + /// the adapter derives (as `upsert`/`delete` now do) misses rows this + /// double filed under an invented tag — which is a bug in the double, not + /// in the adapter. + tag: String, } /// The doubles' shared store: `id -> Row`, plus a counter for fresh ids. @@ -105,6 +112,9 @@ async fn mem0_create(State(store): State, Json(body): Json) -> Jso id: id.clone(), content, metadata, + // Mem0 has no container tags; rows carry an empty one and the + // supermemory-only tag routes never see them. + tag: String::new(), }, ); Json(json!({ "results": [{ "id": id }] })) @@ -196,16 +206,19 @@ async fn the_mem0_double_actually_retains() { /// The tag the adapter derives, as sent on create. fn tag_of(row: &Row) -> String { - row.metadata - .get("tinymemory_namespace") - .and_then(Value::as_str) - .map(|ns| format!("tinymemory-{ns}")) - .unwrap_or_default() + row.tag.clone() } async fn sm_tags(State(store): State) -> Json { let store = store.lock().expect("store lock"); - let mut tags: Vec = store.rows.values().map(tag_of).collect(); + // Mem0 rows carry an empty tag (that dialect has no containers); they must + // not surface as a Supermemory container. + let mut tags: Vec = store + .rows + .values() + .map(tag_of) + .filter(|tag| !tag.is_empty()) + .collect(); tags.sort(); tags.dedup(); Json(Value::Array( @@ -243,7 +256,15 @@ async fn sm_list(State(store): State, Json(body): Json) -> Json, Json(body): Json) -> Json { +async fn sm_create( + State(store): State, + Json(body): Json, +) -> Result, axum::http::StatusCode> { + // The real v4 API requires `containerTag`; a double that silently filed a + // malformed create under "" would hide an adapter regression. + let Some(tag) = body["containerTag"].as_str().filter(|tag| !tag.is_empty()) else { + return Err(axum::http::StatusCode::BAD_REQUEST); + }; let mut store = store.lock().expect("store lock"); let id = store.fresh_id(); let first = &body["memories"][0]; @@ -253,9 +274,10 @@ async fn sm_create(State(store): State, Json(body): Json) -> Json< id: id.clone(), content: first["content"].as_str().unwrap_or_default().to_owned(), metadata: first["metadata"].clone(), + tag: tag.to_owned(), }, ); - Json(json!({ "memories": [{ "id": id }] })) + Ok(Json(json!({ "memories": [{ "id": id }] }))) } async fn sm_update(State(store): State, Json(body): Json) -> Json { diff --git a/adapters/remote/src/supermemory.rs b/adapters/remote/src/supermemory.rs index d1c70d0..633fa76 100644 --- a/adapters/remote/src/supermemory.rs +++ b/adapters/remote/src/supermemory.rs @@ -224,6 +224,20 @@ impl SupermemoryDialect { } let mut entries = Vec::new(); for container_tag in container_tags { + entries.extend(self.memories_in_tag(container_tag).await?); + } + Ok(entries) + } + + /// Enumerates the live memories of one container tag. + /// + /// Split out so the keyed paths (`upsert`, `delete`) can page the single + /// tag their namespace maps to instead of every tag the account holds — + /// before this, each store of one record enumerated the entire account + /// over HTTP. + async fn memories_in_tag(&self, container_tag: &str) -> anyhow::Result> { + let mut entries = Vec::new(); + { let mut page = 1_u64; loop { let response: Value = self @@ -273,6 +287,16 @@ impl SupermemoryDialect { } Ok(entries) } + + /// The live entry stored under `(namespace, key)`, if any — paging only + /// that namespace's container tag. + async fn find_entry(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .memories_in_tag(&Self::container_tag(namespace)) + .await? + .into_iter() + .find(|item| item.namespace == namespace && item.key == key)) + } } #[async_trait] @@ -284,11 +308,7 @@ impl Dialect for SupermemoryDialect { /// Replaces an existing exact record or creates a direct v4 memory. async fn upsert(&self, entry: StoredEntry) -> anyhow::Result<()> { - let existing = self - .memories() - .await? - .into_iter() - .find(|item| item.namespace == entry.namespace && item.key == entry.key); + let existing = self.find_entry(&entry.namespace, &entry.key).await?; let metadata = Self::metadata(&entry); if let Some(existing) = existing { self.client @@ -361,12 +381,7 @@ impl Dialect for SupermemoryDialect { /// Finds and deletes an exact TinyMemory logical record. async fn delete(&self, namespace: &str, key: &str) -> anyhow::Result { - let Some(entry) = self - .memories() - .await? - .into_iter() - .find(|item| item.namespace == namespace && item.key == key) - else { + let Some(entry) = self.find_entry(namespace, key).await? else { return Ok(false); }; self.client diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index 57ae0bf..c39933e 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -64,6 +64,19 @@ use tinymemory_api::mandatory::MemoryTraitProvider; /// this adapter out still refuses to bind something else under the name. pub use tinymemory_api::drivers::TINYCORTEX_DRIVER_ID; +/// The engine crate itself, re-exported so a consumer of this adapter can +/// name the [`tinycortex::memory::Memory`] argument type and construct a +/// backend without adding a second git dependency and its `[patch]` table. +/// `tinymemory::tinycortex::provider(...)` was unusable from outside this +/// workspace before this line: the feature compiled, the constructor +/// resolved, and its argument type was unnameable. +pub use tinycortex; + +/// The engine's simplest backend, re-exported for first-run and test wiring: +/// `provider(Arc::new(InMemoryMemoryStore::new()))` is a complete embedded +/// setup for the mandatory three families. +pub use tinycortex::memory::store::InMemoryMemoryStore; + /// Wrap a TinyCortex backend as a bound memory driver. /// /// The returned provider advertises the mandatory three families and nothing diff --git a/api/src/lib.rs b/api/src/lib.rs index bf237aa..cb7d52c 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -41,10 +41,10 @@ //! - [`recall`]: the borrowed [`recall::RecallOpts`] and owned, serde-derived //! [`recall::OwnedRecallOpts`] recall filters (both re-exported from //! [`types`]). -//! - [`capabilities`]: the sixteen [`capabilities::Capability`] families and +//! - [`capabilities`]: the eighteen [`capabilities::Capability`] families and //! the [`capabilities::Capabilities`] set negotiated at bind time. //! - [`provider`]: the driver contract — [`provider::MemoryProvider`] plus the -//! sixteen capability family traits and the value types they need. +//! eighteen capability family traits and the value types they need. //! - [`null`]: [`null::NullMemoryProvider`], the reference driver a //! compiled-out or unconfigured memory subsystem binds to. //! - [`health`]: [`health::MemoryHealth`], the liveness state a driver reports. diff --git a/examples/tinycortex.rs b/examples/tinycortex.rs new file mode 100644 index 0000000..fe4b1ea --- /dev/null +++ b/examples/tinycortex.rs @@ -0,0 +1,63 @@ +//! The embedded engine, end to end: admit, construct, audit, store, recall. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example tinycortex --features tinycortex +//! ``` +//! +//! `examples/basic.rs` teaches the binding *shape* with the null driver; this +//! one proves the first real engine binds the same way and actually retains. +//! The backend is the engine's own in-memory store — a complete embedded +//! setup for the mandatory three families: no workspace, no host seams. (The +//! full eighteen-family `TinycortexProvider` additionally needs the host +//! seams installed; `adapters/tinycortex/tests/full_provider_conformance.rs` +//! is the minimal working wiring for that.) + +use std::sync::Arc; + +use tinymemory::api::provider::{audit_provider, MemoryProvider}; +use tinymemory::api::recall::OwnedRecallOpts; +use tinymemory::api::types::{MemoryCategory, MemoryTaint}; +use tinymemory::registry::{ConfigLabels, DriverRegistry, TINYCORTEX_DRIVER_ID}; +use tinymemory::tinycortex::{provider, InMemoryMemoryStore}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // 1. Admission first: is the id real, and may it answer for memory. + let registry = DriverRegistry::builtin(); + let admission = registry.admit(TINYCORTEX_DRIVER_ID, None, ConfigLabels::default())?; + println!("admitted '{}' as {:?}", admission.id, admission.class); + + // 2. Construction: the engine's simplest backend, wrapped as a driver. + let provider: Arc = + Arc::new(provider(Arc::new(InMemoryMemoryStore::new()))); + + // 3. The capability audit: advertised must equal reachable. + audit_provider(provider.as_ref())?; + println!( + "driver '{}' serves {} families", + provider.driver_id(), + provider.capabilities().iter().count() + ); + + // 4. Store and recall through the contract — no engine type in sight. + provider + .store( + "example", + "greeting", + "the embedded engine says hello", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await?; + let opts = OwnedRecallOpts { + namespace: Some("example".into()), + ..OwnedRecallOpts::default() + }; + let hits = provider.recall("hello", 8, &opts, None).await?; + println!("recall found {} entr(y/ies)", hits.len()); + assert!(!hits.is_empty(), "the stored entry must be recallable"); + Ok(()) +}