diff --git a/CHANGELOG.md b/CHANGELOG.md index 9632f8e..8bebef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to ShellClaw are documented here. Format follows [Keep a Cha ## [Unreleased] ### Fixed +- Session compaction includes the boundary turn (index `msg_count - max_context_messages`) in the summary window, so the turn at the keep-tail boundary is not deleted without being summarized. +- Session compaction keeps a message that does not fit the 64 KiB summary source, and everything after it, instead of skipping the long line and deleting it anyway. +- Session compaction no longer persists the summarized transcript before the user turn finishes: a failed follow-up provider call leaves the raw stored session unchanged, and when the kept tail already starts with a user turn the summary is folded into that turn so consecutive user roles cannot break the next Anthropic call (supersedes #126). +- Session compaction summarizes a message larger than the summary source as a truncated `[truncated]` stub, so a single oversized line can no longer abort the pass with an empty summary source or keep the session above `max_context_messages` forever. - WebChat `ws_send_to` drops a send whose session has no live `wsi` (still returning 0) and `ws_unregister_conn` drops frames still queued for the closing session, so ~64 abandoned replies cannot pin the shared outgoing queue at `MAX_QUEUE` (64) and stall every live client until restart. - `read_file` reads that do not fit the result buffer now fail with a JSON error instead of returning a clipped prefix, so a later `write_file` cannot persist a silently truncated file. - Cron re-offer waits the full timeout when a due id could not be stored in the 16-slot table, and an ack frees that slot so the next due job can be delivered and then backed off. diff --git a/src/core/agent.c b/src/core/agent.c index 92bc6f0..22d9d2e 100644 --- a/src/core/agent.c +++ b/src/core/agent.c @@ -82,7 +82,145 @@ static int copy_printed_session_json(char *dst, size_t dst_size, char *printed, static const char SUMMARIZE_SYSTEM[] = "Summarize the following conversation in one short paragraph. Output only the summary, no preamble."; -/** Summarize oldest messages when over max_ctx; replace with one summary + trailing. */ +/** Marker appended by summary_source_append_truncated when a line was cut. */ +static const char SUMMARY_TRUNC_MARKER[] = "[truncated]"; + +/** Append one "role: content" line, or roll back if the whole line does not fit. */ +static int summary_source_append(char *buf, size_t cap, size_t *used, + const char *role, const char *content) +{ + int n; + if (!buf || !used || *used >= cap - 1) + return 0; + n = snprintf(buf + *used, cap - *used, "%s: %s\n", + role ? role : "user", content ? content : ""); + if (n < 0 || (size_t)n >= cap - *used) { + buf[*used] = '\0'; + return 0; + } + *used += (size_t)n; + return 1; +} + +/** True when the line cannot fit even a fresh source, so no later pass can + * ever summarize it whole either. */ +static int summary_line_never_fits(const char *role, const char *content) +{ + const char *r = role ? role : "user"; + const char *c = content ? content : ""; + + /* summary_source_append refuses when the line plus its NUL exceeds the cap. */ + return strlen(r) + 2 + strlen(c) + 1 >= SUMMARY_SOURCE_MAX; +} + +/** Append one line with the content cut to fit, marking the cut, so a message + * that can never fit whole still reaches the summarizer. Same atomicity: + * the whole stub fits or nothing is written. */ +static int summary_source_append_truncated(char *buf, size_t cap, size_t *used, + const char *role, const char *content) +{ + const char *r = role ? role : "user"; + const char *c = content ? content : ""; + size_t remain; + size_t fixed; + size_t cut; + int n; + + if (!buf || !used || *used + 1 >= cap) + return 0; + remain = cap - *used; + fixed = strlen(r) + 2 + strlen(SUMMARY_TRUNC_MARKER) + 1 + 1; /* ": " "\n" NUL */ + if (remain <= fixed) + return 0; + cut = remain - fixed; + if (cut > strlen(c)) + cut = strlen(c); + n = snprintf(buf + *used, remain, "%s: %.*s%s\n", r, (int)cut, c, + SUMMARY_TRUNC_MARKER); + if (n < 0 || (size_t)n >= remain) { + buf[*used] = '\0'; + return 0; + } + *used += (size_t)n; + return 1; +} + +/** True when the kept tail already starts with user, so a summary user turn would not alternate. */ +static int kept_tail_starts_with_user(const cJSON *root, int start) +{ + const cJSON *item; + const cJSON *role; + + item = cJSON_GetArrayItem(root, start); + if (!item || !cJSON_IsObject(item)) + return 0; + role = cJSON_GetObjectItem(item, "role"); + return cJSON_IsString(role) && role->valuestring && strcmp(role->valuestring, "user") == 0; +} + +/** Fold the summary into an existing user turn. A second user message 400s on Anthropic. */ +static int fold_summary_into_user(cJSON *item, const char *summary) +{ + const cJSON *content; + const char *old; + char *merged; + cJSON *node; + size_t old_len; + size_t sum_len; + size_t n; + + content = cJSON_GetObjectItem(item, "content"); + old = (content && cJSON_IsString(content) && content->valuestring) ? content->valuestring : ""; + old_len = strlen(old); + sum_len = strlen(summary ? summary : ""); + if (sum_len > SIZE_MAX - old_len - 3) + return -1; + n = sum_len + 2 + old_len + 1; + merged = malloc(n); + if (!merged) + return -1; + snprintf(merged, n, "%s\n\n%s", summary ? summary : "", old); + node = cJSON_CreateString(merged); + free(merged); + if (!node) + return -1; + if (cJSON_GetObjectItem(item, "content")) { + if (!cJSON_ReplaceItemInObject(item, "content", node)) { + cJSON_Delete(node); + return -1; + } + return 0; + } + if (!cJSON_AddItemToObject(item, "content", node)) { + cJSON_Delete(node); + return -1; + } + return 0; +} + +static int append_summary_user(cJSON *arr, const char *summary) +{ + cJSON *obj; + cJSON *role; + cJSON *content; + + obj = cJSON_CreateObject(); + role = cJSON_CreateString("user"); + content = cJSON_CreateString(summary ? summary : ""); + if (!obj || !role || !content) { + cJSON_Delete(obj); + cJSON_Delete(role); + cJSON_Delete(content); + return -1; + } + cJSON_AddItemToObject(obj, "role", role); + cJSON_AddItemToObject(obj, "content", content); + cJSON_AddItemToArray(arr, obj); + return 0; +} + +/** Summarize oldest messages when over max_ctx; replace with one summary + trailing. + * Messages that do not fit in the summary source stay in the session. */ static int compact_session_via_llm(const char *session_id, char *session_buf, size_t session_buf_size, int msg_count, int max_ctx, const provider_t *provider) { @@ -92,22 +230,42 @@ static int compact_session_via_llm(const char *session_id, char *session_buf, si if (root) cJSON_Delete(root); return -1; } - int to_summarize_n = msg_count - max_ctx; + /* Tail keeps max_ctx-1 messages, so the summary must include index + * msg_count-max_ctx. Stopping one short drops that turn from SQLite. */ + int to_summarize_n = msg_count - max_ctx + 1; char *source_buf = malloc(SUMMARY_SOURCE_MAX); if (!source_buf) { cJSON_Delete(root); return -1; } source_buf[0] = '\0'; size_t used = 0; - for (int i = 0; i < to_summarize_n && used < SUMMARY_SOURCE_MAX - 1; i++) { + int next_unsummarized = to_summarize_n; + for (int i = 0; i < to_summarize_n; i++) { const cJSON *item = cJSON_GetArrayItem(root, i); - if (!item || !cJSON_IsObject(item)) continue; const char *role = "user"; const char *content = ""; - const cJSON *r = cJSON_GetObjectItem(item, "role"); - const cJSON *c = cJSON_GetObjectItem(item, "content"); + const cJSON *r; + const cJSON *c; + if (!item || !cJSON_IsObject(item)) continue; + r = cJSON_GetObjectItem(item, "role"); + c = cJSON_GetObjectItem(item, "content"); if (cJSON_IsString(r)) role = r->valuestring; if (cJSON_IsString(c)) content = c->valuestring; - int n = snprintf(source_buf + used, SUMMARY_SOURCE_MAX - used, "%s: %s\n", role, content ? content : ""); - if (n > 0 && (size_t)n < SUMMARY_SOURCE_MAX - used) used += (size_t)n; + if (!summary_source_append(source_buf, SUMMARY_SOURCE_MAX, &used, role, content)) { + /* A line that can never fit whole would abort with used==0 + * (first line) or keep the session above max_ctx forever + * (later line); stub it so this pass still consumes it. + * Lines that only lack room here stay raw for a later pass + * via next_unsummarized. */ + if (!summary_line_never_fits(role, content) || + !summary_source_append_truncated(source_buf, SUMMARY_SOURCE_MAX, &used, role, content)) { + next_unsummarized = i; + break; + } + } + } + if (used == 0) { + free(source_buf); + cJSON_Delete(root); + return -1; } provider_message_t summ_msgs[2] = { { .role = "system", .content = SUMMARIZE_SYSTEM, .tool_calls = NULL, .tool_calls_count = 0, .tool_use_id = NULL }, @@ -128,29 +286,51 @@ static int compact_session_via_llm(const char *session_id, char *session_buf, si summary_buf[sum_len] = '\0'; provider_response_clear(&resp); cJSON *new_arr = cJSON_CreateArray(); - if (!new_arr) { cJSON_Delete(root); return -1; } - cJSON *summ_obj = cJSON_CreateObject(); - if (summ_obj) { - cJSON_AddItemToObject(summ_obj, "role", cJSON_CreateString("user")); - cJSON_AddItemToObject(summ_obj, "content", cJSON_CreateString(summary_buf)); - cJSON_AddItemToArray(new_arr, summ_obj); - } int keep = max_ctx - 1; - int start = msg_count - keep; + int start = (next_unsummarized < to_summarize_n) + ? next_unsummarized : (msg_count - keep); + int merge; + int folded = 0; if (start < 0) start = 0; + /* Tail can start on a user turn (odd max_context_messages, or a cut that + * lands on user). Inserting another user role makes Anthropic reject the + * session on this turn and every later one. */ + merge = kept_tail_starts_with_user(root, start); + if (!new_arr) { cJSON_Delete(root); return -1; } + if (!merge && append_summary_user(new_arr, summary_buf) != 0) { + cJSON_Delete(new_arr); + cJSON_Delete(root); + return -1; + } for (int i = start; i < msg_count; i++) { const cJSON *item = cJSON_GetArrayItem(root, i); - if (item) { - cJSON *dup = cJSON_Duplicate(item, 1); - if (dup) cJSON_AddItemToArray(new_arr, dup); + cJSON *dup; + if (!item) continue; + dup = cJSON_Duplicate(item, 1); + if (!dup) continue; + if (merge && i == start) { + if (fold_summary_into_user(dup, summary_buf) != 0) { + cJSON_Delete(dup); + cJSON_Delete(new_arr); + cJSON_Delete(root); + return -1; + } + folded = 1; } + cJSON_AddItemToArray(new_arr, dup); + } + if (merge && !folded) { + cJSON_Delete(new_arr); + cJSON_Delete(root); + return -1; } cJSON_Delete(root); char *printed = cJSON_PrintUnformatted(new_arr); cJSON_Delete(new_arr); if (copy_printed_session_json(session_buf, session_buf_size, printed, session_id) != 0) return -1; - session_save(session_id, session_buf); + /* Do not session_save here. A failed follow-up chat must leave the stored + * transcript unchanged; agent_persist_session writes this buffer on success. */ return 0; } diff --git a/tests/test_agent.c b/tests/test_agent.c index 4b3b7e6..e1e9c22 100644 --- a/tests/test_agent.c +++ b/tests/test_agent.c @@ -366,7 +366,16 @@ static int test_session_persisted_after_exchange(void) } static int compaction_call_count; -static int compaction_init(const config_t *cfg) { (void)cfg; compaction_call_count = 0; return 0; } +static int compaction_source_has_boundary; +static int compaction_source_truncated; +static int compaction_init(const config_t *cfg) +{ + (void)cfg; + compaction_call_count = 0; + compaction_source_has_boundary = 0; + compaction_source_truncated = 0; + return 0; +} static int compaction_chat(const provider_message_t *messages, size_t message_count, const provider_tool_def_t *tools, size_t tool_count, provider_response_t *response) { @@ -377,6 +386,11 @@ static int compaction_chat(const provider_message_t *messages, size_t message_co response->tool_calls_count = 0; compaction_call_count++; if (compaction_call_count == 1) { + const char *src = (message_count >= 2 && messages[1].content) ? messages[1].content : ""; + /* msg_10 is the boundary turn: the summary window must include it; + * the kept tail starts at msg_11. */ + compaction_source_has_boundary = strstr(src, "msg_10\n") != NULL; + compaction_source_truncated = strstr(src, "[truncated]") != NULL; response->content = strdup("Summary of earlier conversation."); return 0; } @@ -442,9 +456,553 @@ static int test_context_compaction_when_history_exceeds_max(void) } if (!found_summary) goto cleanup; if (!found_tail) goto cleanup; + /* Boundary message must be in the summary source; the tail starts at msg_11. */ + if (!compaction_source_has_boundary) goto cleanup; + failed = 0; +cleanup: + config_free(cfg); +cleanup_early: + remove(config_path); + remove(db_path); + memory_cleanup(); + return failed; +} + +/* A message that fits a fresh source but not the space left by earlier lines + * must stay in the session. Deleting it loses history the model never saw. */ +static int test_compaction_keeps_message_that_does_not_fit_summary(void) +{ + enum { HEAD = 30 * 1024, BIG = 5 * 1024, JSON_CAP = 96 * 1024, LOAD_CAP = 128 * 1024 }; + int failed = 1; + config_t *cfg = NULL; + char *session_json = NULL; + char *loaded = NULL; + char *head_a = NULL; + char *head_b = NULL; + char *big = NULL; + const char *db_path = "build/test_agent_compact_fit.db"; + const char *config_path = "build/test_agent_compact_fit.toml"; + size_t off; + int i; + char response_buf[4096]; + char errbuf[256]; + FILE *cf; + + memory_cleanup(); + compaction_call_count = 0; + if (memory_init(db_path) != 0) + goto cleanup_early; + session_json = malloc(JSON_CAP); + head_a = malloc((size_t)HEAD + 1); + head_b = malloc((size_t)HEAD + 1); + big = malloc((size_t)BIG + 1); + loaded = malloc(LOAD_CAP); + if (!session_json || !head_a || !head_b || !big || !loaded) + goto cleanup; + memset(head_a, 'a', (size_t)HEAD); + memcpy(head_a + HEAD - 10, "HEAD_A_END", 10); + head_a[HEAD] = '\0'; + memset(head_b, 'b', (size_t)HEAD); + memcpy(head_b + HEAD - 10, "HEAD_B_END", 10); + head_b[HEAD] = '\0'; + memset(big, 'A', (size_t)BIG); + memcpy(big + BIG - 12, "BIG_KEEP_END", 12); + big[BIG] = '\0'; + off = 0; + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "["); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + "{\"role\":\"user\",\"content\":\"%s\"}", head_a); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"%s\"}", head_b); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"%s\"}", big); + for (i = 3; i < 6; i++) { + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"m%d\"}", i); + } + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "]"); + if (off >= JSON_CAP) + goto cleanup; + if (session_save("cli:compactfit", session_json) != 0) + goto cleanup; + cf = fopen(config_path, "w"); + if (!cf) + goto cleanup; + fprintf(cf, "[agent]\nmodel = \"test\"\nmax_context_messages = 4\n[memory]\npath = \"%s\"\n", + db_path); + fclose(cf); + if (config_load(config_path, &cfg, errbuf, sizeof(errbuf)) != 0) + goto cleanup; + response_buf[0] = '\0'; + if (agent_run(cfg, "cli:compactfit", "hi", &compaction_provider, NULL, 0, + response_buf, sizeof(response_buf)) != 0) + goto cleanup; + if (session_load("cli:compactfit", loaded, LOAD_CAP) != 0) + goto cleanup; + if (strstr(loaded, "BIG_KEEP_END") == NULL) { + fprintf(stderr, + "FAIL: compacted session dropped the message that did not fit the summary\n"); + goto cleanup; + } + if (strstr(loaded, "Summary of earlier") == NULL) { + fprintf(stderr, "FAIL: expected a summary of the messages that did fit\n"); + goto cleanup; + } + if (strstr(loaded, "HEAD_A_END") != NULL || strstr(loaded, "HEAD_B_END") != NULL) { + fprintf(stderr, "FAIL: head messages that fit the summary were kept raw\n"); + goto cleanup; + } + failed = 0; +cleanup: + config_free(cfg); + free(session_json); + free(head_a); + free(head_b); + free(big); + free(loaded); +cleanup_early: + remove(config_path); + remove(db_path); + memory_cleanup(); + return failed; +} + +static int count_session_messages(const char *json) +{ + cJSON *root; + int n; + + root = cJSON_Parse(json); + if (!root || !cJSON_IsArray(root)) { + if (root) + cJSON_Delete(root); + return -1; + } + n = cJSON_GetArraySize(root); + cJSON_Delete(root); + return n; +} + +/* An oversized FIRST line used to abort compaction with an empty source, so + * the session never shrank again and every turn retried in vain. The line now + * reaches the summarizer as a truncated stub and the session keeps shrinking. */ +static int test_compaction_progresses_when_first_message_is_oversized(void) +{ + enum { OVERSIZE = 65 * 1024, JSON_CAP = 96 * 1024, LOAD_CAP = 128 * 1024 }; + int failed = 1; + config_t *cfg = NULL; + char *session_json = NULL; + char *loaded = NULL; + char *big = NULL; + const char *db_path = "build/test_agent_compact_stall_first.db"; + const char *config_path = "build/test_agent_compact_stall_first.toml"; + size_t off; + int i; + char response_buf[4096]; + char errbuf[256]; + FILE *cf; + + memory_cleanup(); + compaction_call_count = 0; + if (memory_init(db_path) != 0) + goto cleanup_early; + session_json = malloc(JSON_CAP); + big = malloc((size_t)OVERSIZE + 1); + loaded = malloc(LOAD_CAP); + if (!session_json || !big || !loaded) + goto cleanup; + memset(big, 'A', (size_t)OVERSIZE); + memcpy(big + OVERSIZE - 11, "STALL_A_END", 11); + big[OVERSIZE] = '\0'; + off = 0; + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "["); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + "{\"role\":\"user\",\"content\":\"%s\"}", big); + for (i = 1; i < 6; i++) { + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"m%d\"}", i); + } + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "]"); + if (off >= JSON_CAP) + goto cleanup; + if (session_save("cli:stallfirst", session_json) != 0) + goto cleanup; + cf = fopen(config_path, "w"); + if (!cf) + goto cleanup; + fprintf(cf, "[agent]\nmodel = \"test\"\nmax_context_messages = 4\n[memory]\npath = \"%s\"\n", + db_path); + fclose(cf); + if (config_load(config_path, &cfg, errbuf, sizeof(errbuf)) != 0) + goto cleanup; + response_buf[0] = '\0'; + if (agent_run(cfg, "cli:stallfirst", "hi", &compaction_provider, NULL, 0, + response_buf, sizeof(response_buf)) != 0) + goto cleanup; + if (compaction_call_count < 2) { + fprintf(stderr, "FAIL: compaction aborted before the summarizer call\n"); + goto cleanup; + } + if (!compaction_source_truncated) { + fprintf(stderr, + "FAIL: oversized first line did not reach the summary as a truncated stub\n"); + goto cleanup; + } + if (session_load("cli:stallfirst", loaded, LOAD_CAP) != 0) + goto cleanup; + if (strstr(loaded, "STALL_A_END") != NULL) { + fprintf(stderr, "FAIL: oversized first line is still raw instead of summarized\n"); + goto cleanup; + } + if (strstr(loaded, "Summary of earlier") == NULL) { + fprintf(stderr, "FAIL: expected a summary of the truncated oversized turn\n"); + goto cleanup; + } + /* A follow-up turn must finish shrinking the session down to max_ctx. */ + if (agent_run(cfg, "cli:stallfirst", "hi", &compaction_provider, NULL, 0, + response_buf, sizeof(response_buf)) != 0) + goto cleanup; + if (session_load("cli:stallfirst", loaded, LOAD_CAP) != 0) + goto cleanup; + if (count_session_messages(loaded) > 4 + 2) { + fprintf(stderr, "FAIL: session never shrinks back to max_ctx after the oversized turn\n"); + goto cleanup; + } + failed = 0; +cleanup: + config_free(cfg); + free(session_json); + free(big); + free(loaded); +cleanup_early: + remove(config_path); + remove(db_path); + memory_cleanup(); + return failed; +} + +/* An oversized LATER line used to be kept raw with everything after it, so + * new_count stayed above max_ctx forever. The truncated stub now consumes it + * and this pass brings the session back down to max_ctx. */ +static int test_compaction_progresses_when_later_message_is_oversized(void) +{ + enum { OVERSIZE = 65 * 1024, JSON_CAP = 96 * 1024, LOAD_CAP = 128 * 1024 }; + int failed = 1; + config_t *cfg = NULL; + char *session_json = NULL; + char *loaded = NULL; + char *big = NULL; + const char *db_path = "build/test_agent_compact_stall_later.db"; + const char *config_path = "build/test_agent_compact_stall_later.toml"; + size_t off; + int i; + char response_buf[4096]; + char errbuf[256]; + FILE *cf; + + memory_cleanup(); + compaction_call_count = 0; + if (memory_init(db_path) != 0) + goto cleanup_early; + session_json = malloc(JSON_CAP); + big = malloc((size_t)OVERSIZE + 1); + loaded = malloc(LOAD_CAP); + if (!session_json || !big || !loaded) + goto cleanup; + memset(big, 'A', (size_t)OVERSIZE); + memcpy(big + OVERSIZE - 11, "STALL_B_END", 11); + big[OVERSIZE] = '\0'; + off = 0; + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "["); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + "{\"role\":\"user\",\"content\":\"alpha_small\"}"); + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"%s\"}", big); + for (i = 2; i < 6; i++) { + off += (size_t)snprintf(session_json + off, JSON_CAP - off, + ",{\"role\":\"user\",\"content\":\"m%d\"}", i); + } + off += (size_t)snprintf(session_json + off, JSON_CAP - off, "]"); + if (off >= JSON_CAP) + goto cleanup; + if (session_save("cli:stalllater", session_json) != 0) + goto cleanup; + cf = fopen(config_path, "w"); + if (!cf) + goto cleanup; + fprintf(cf, "[agent]\nmodel = \"test\"\nmax_context_messages = 4\n[memory]\npath = \"%s\"\n", + db_path); + fclose(cf); + if (config_load(config_path, &cfg, errbuf, sizeof(errbuf)) != 0) + goto cleanup; + response_buf[0] = '\0'; + if (agent_run(cfg, "cli:stalllater", "hi", &compaction_provider, NULL, 0, + response_buf, sizeof(response_buf)) != 0) + goto cleanup; + if (!compaction_source_truncated) { + fprintf(stderr, + "FAIL: oversized later line did not reach the summary as a truncated stub\n"); + goto cleanup; + } + if (session_load("cli:stalllater", loaded, LOAD_CAP) != 0) + goto cleanup; + if (strstr(loaded, "STALL_B_END") != NULL) { + fprintf(stderr, "FAIL: oversized later line is still raw instead of summarized\n"); + goto cleanup; + } + if (strstr(loaded, "Summary of earlier") == NULL) { + fprintf(stderr, "FAIL: expected a summary of the truncated oversized turn\n"); + goto cleanup; + } + if (count_session_messages(loaded) > 4 + 2) { + fprintf(stderr, "FAIL: session is not back to max_ctx after the oversized turn\n"); + goto cleanup; + } + failed = 0; +cleanup: + config_free(cfg); + free(session_json); + free(big); + free(loaded); +cleanup_early: + remove(config_path); + remove(db_path); + memory_cleanup(); + return failed; +} + +static int save_alternating_session(const char *session_id, int count) +{ + char *json; + size_t cap; + size_t off = 0; + int i; + int rc; + + cap = (size_t)count * 80u + 2u; + json = malloc(cap); + if (!json) + return -1; + off += (size_t)snprintf(json + off, cap - off, "["); + for (i = 0; i < count; i++) { + const char *role = (i % 2 == 0) ? "user" : "assistant"; + int n = snprintf(json + off, cap - off, + "%s{\"role\":\"%s\",\"content\":\"msg_%d\"}", + i ? "," : "", role, i); + if (n < 0 || (size_t)n >= cap - off) { + free(json); + return -1; + } + off += (size_t)n; + } + if (off + 1 >= cap) { + free(json); + return -1; + } + json[off++] = ']'; + json[off] = '\0'; + rc = session_save(session_id, json); + free(json); + return rc; +} + +static int json_roles_alternate_from_user(const char *json) +{ + cJSON *root; + int n; + int i; + const char *prev = NULL; + + root = cJSON_Parse(json); + if (!root || !cJSON_IsArray(root)) { + if (root) + cJSON_Delete(root); + return 0; + } + n = cJSON_GetArraySize(root); + if (n <= 0) { + cJSON_Delete(root); + return 0; + } + for (i = 0; i < n; i++) { + cJSON *item = cJSON_GetArrayItem(root, i); + cJSON *role = item ? cJSON_GetObjectItem(item, "role") : NULL; + const char *r = (role && cJSON_IsString(role) && role->valuestring) ? role->valuestring : ""; + if (i == 0 && strcmp(r, "user") != 0) { + cJSON_Delete(root); + return 0; + } + if (prev && strcmp(prev, r) == 0) { + cJSON_Delete(root); + return 0; + } + prev = r; + } + cJSON_Delete(root); + return 1; +} + +static int compact_fail_calls; +static int compact_fail_init(const config_t *cfg) +{ + (void)cfg; + compact_fail_calls = 0; + return 0; +} +static int compact_fail_chat(const provider_message_t *messages, size_t message_count, + const provider_tool_def_t *tools, size_t tool_count, provider_response_t *response) +{ + (void)messages; + (void)message_count; + (void)tools; + (void)tool_count; + response->tool_calls = NULL; + response->tool_calls_count = 0; + response->error = 0; + compact_fail_calls++; + if (compact_fail_calls == 1) { + response->content = strdup("Summary of earlier conversation."); + return 0; + } + provider_set_error(response, "mock provider error"); + return -1; +} +static void compact_fail_cleanup(void) {} +static const provider_t compact_fail_provider = { + .name = "compact_fail", + .init = compact_fail_init, + .chat = compact_fail_chat, + .cleanup = compact_fail_cleanup, +}; + +/* Summary is computed before the user turn. Saving it immediately deletes the + * raw transcript when the follow-up chat fails and the turn is not stored. */ +static int test_compaction_failure_keeps_original_session(void) +{ + int failed = 1; + config_t *cfg = NULL; + char *loaded = NULL; + const char *db_path = "build/test_agent_compact_fail.db"; + const char *config_path = "build/test_agent_compact_fail.toml"; + char response_buf[4096]; + char errbuf[256]; + FILE *cf; + + memory_cleanup(); + compact_fail_calls = 0; + if (memory_init(db_path) != 0) + goto cleanup_early; + if (save_alternating_session("cli:compactfail", 50) != 0) + goto cleanup; + cf = fopen(config_path, "w"); + if (!cf) + goto cleanup; + fprintf(cf, "[agent]\nmodel = \"test\"\nmax_context_messages = 40\n[memory]\npath = \"%s\"\n", + db_path); + fclose(cf); + if (config_load(config_path, &cfg, errbuf, sizeof(errbuf)) != 0) + goto cleanup; + loaded = malloc(128 * 1024); + if (!loaded) + goto cleanup; + response_buf[0] = '\0'; + if (agent_run(cfg, "cli:compactfail", "hi", &compact_fail_provider, NULL, 0, + response_buf, sizeof(response_buf)) == 0) + goto cleanup; + if (strstr(response_buf, "mock provider error") == NULL) + goto cleanup; + if (session_load("cli:compactfail", loaded, 128 * 1024) != 0) + goto cleanup; + if (strstr(loaded, "\"content\":\"msg_0\"") == NULL) { + fprintf(stderr, "FAIL: failed turn replaced the stored transcript\n"); + goto cleanup; + } + if (strstr(loaded, "Summary of earlier") != NULL) { + fprintf(stderr, "FAIL: summary was saved even though the turn failed\n"); + goto cleanup; + } + failed = 0; +cleanup: + config_free(cfg); + free(loaded); +cleanup_early: + remove(config_path); + remove(db_path); + memory_cleanup(); + return failed; +} + +/* Odd max_context_messages cuts the tail on a user turn. A separate summary + * user message then makes every later Anthropic call a 400. */ +static int test_compaction_merges_summary_when_tail_starts_with_user(void) +{ + int failed = 1; + config_t *cfg = NULL; + char *loaded = NULL; + const char *db_path = "build/test_agent_compact_roles.db"; + const char *config_path = "build/test_agent_compact_roles.toml"; + char response_buf[4096]; + char errbuf[256]; + FILE *cf; + size_t i; + + memory_cleanup(); + compaction_call_count = 0; + if (memory_init(db_path) != 0) + goto cleanup_early; + if (save_alternating_session("cli:compactroles", 8) != 0) + goto cleanup; + cf = fopen(config_path, "w"); + if (!cf) + goto cleanup; + fprintf(cf, "[agent]\nmodel = \"test\"\nmax_context_messages = 5\n[memory]\npath = \"%s\"\n", + db_path); + fclose(cf); + if (config_load(config_path, &cfg, errbuf, sizeof(errbuf)) != 0) + goto cleanup; + loaded = malloc(64 * 1024); + if (!loaded) + goto cleanup; + response_buf[0] = '\0'; + spy_roles_clear(); + if (agent_run(cfg, "cli:compactroles", "hi", &compaction_provider, NULL, 0, + response_buf, sizeof(response_buf)) != 0) + goto cleanup; + if (compaction_call_count < 2) + goto cleanup; + if (spy_message_count < 6 || spy_message_count > SPY_SLOTS) + goto cleanup; + if (!spy_roles[0] || strcmp(spy_roles[0], "system") != 0) + goto cleanup; + if (!spy_roles[1] || strcmp(spy_roles[1], "user") != 0) + goto cleanup; + if (strstr(spy_content[1], "Summary of earlier") == NULL || + strstr(spy_content[1], "msg_4") == NULL) { + fprintf(stderr, "FAIL: summary was not folded into the leading user turn\n"); + goto cleanup; + } + for (i = 2; i < spy_message_count; i++) { + if (!spy_roles[i] || strcmp(spy_roles[i], spy_roles[i - 1]) == 0) { + fprintf(stderr, "FAIL: compacted history has consecutive roles\n"); + goto cleanup; + } + } + if (session_load("cli:compactroles", loaded, 64 * 1024) != 0) + goto cleanup; + if (strstr(loaded, "Summary of earlier") == NULL || strstr(loaded, "msg_4") == NULL) + goto cleanup; + if (strstr(loaded, "\"content\":\"msg_0\"") != NULL) { + fprintf(stderr, "FAIL: successful turn did not persist compaction\n"); + goto cleanup; + } + if (!json_roles_alternate_from_user(loaded)) { + fprintf(stderr, "FAIL: persisted session roles do not alternate\n"); + goto cleanup; + } failed = 0; cleanup: config_free(cfg); + free(loaded); cleanup_early: remove(config_path); remove(db_path); @@ -452,6 +1010,7 @@ static int test_context_compaction_when_history_exceeds_max(void) return failed; } + static int test_local_offline_note_when_active_is_local(void) { const char *path = "build/test_agent_local_note.toml"; @@ -1186,6 +1745,11 @@ int main(void) RUN(test_react_loop_preserves_prior_tool_results()); RUN(test_session_persisted_after_exchange()); RUN(test_context_compaction_when_history_exceeds_max()); + RUN(test_compaction_keeps_message_that_does_not_fit_summary()); + RUN(test_compaction_progresses_when_first_message_is_oversized()); + RUN(test_compaction_progresses_when_later_message_is_oversized()); + RUN(test_compaction_failure_keeps_original_session()); + RUN(test_compaction_merges_summary_when_tail_starts_with_user()); RUN(test_local_offline_note_when_active_is_local()); RUN(test_local_offline_note_skipped_for_non_local()); RUN(test_session_overflow_does_not_corrupt_history());