Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/sentry-allowlisted-logs.md
Comment thread
ramantehlan marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@truefoundry/trueforge': patch
'@truefoundry/trueforge-core': patch
---

Report logged failures to Sentry except HTTP statuses from 200 through 499, including when that status sits on a wrapped cause. The log message is the Sentry title, and the thrown error is the cause and stack. Those events include the active tenant, request, session, and turn ids. Printed lines omit that marker even when Sentry is off.
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,7 @@ export class SessionHandle<
await input.resolver.close().catch((closeError: unknown) => {
input.resolver.logger.warn('TurnResourceResolver.close() failed after createTurn() error', {
err: closeError,
reportable_error: { error: closeError },
});
});
throw error;
Expand Down
5 changes: 4 additions & 1 deletion packages/trueforge-core/src/agent-session/TurnHandle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,10 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
}

await resolver.close().catch((err: unknown) => {
resolver.logger.warn('TurnResourceResolver.close() failed', { err });
resolver.logger.warn('TurnResourceResolver.close() failed', {
err,
reportable_error: { error: err },
});
});

if (!frozenByStore) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,10 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor {
entry.toolCallResult.isStructuredContent = false;
return stored.sandboxCreated;
} catch (error) {
this.logger.error('Failed to upload to sandbox', extractErrorLogFields(error));
this.logger.error('Failed to upload to sandbox', {
...extractErrorLogFields(error),
reportable_error: { error },
});
return undefined;
}
}
Expand Down
3 changes: 2 additions & 1 deletion packages/trueforge-core/src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ export type {
// Errors / utils
export { AgentHarnessError, McpConnectionError, McpDcrConfigurationError } from './errors';
export { REDIS_KEY_NAMESPACE, redisKey } from './redisKeys';
export { describeUnknownError, extractErrorLogFields } from './util/errorLogFields';
export { describeUnknownError, extractErrorLogFields, isReportableError } from './util/errorLogFields';
export {
DEFAULT_MCP_HTTP_BODY_TIMEOUT_MS,
DEFAULT_MCP_HTTP_CONNECT_TIMEOUT_MS,
Expand All @@ -181,6 +181,7 @@ export {
ssrfFetch,
} from './util/outboundFetch';
export { PromiseTimeoutError, withTimeout } from './util/promiseUtils';
export { type ReportableErrorLog } from './util/reportableErrorLog';
export {
assertSafeOutboundUrl,
configureOutboundUrlGuard,
Expand Down
12 changes: 9 additions & 3 deletions packages/trueforge-core/src/core/llm/VercelAILLM.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import type {
} from 'openai/resources/chat';
import type { Logger } from 'winston';
import { ModelProviderError } from '../errors';
import { describeUnknownError, extractErrorLogFields } from '../util/errorLogFields';
import { describeUnknownError, extractErrorLogFields, isReportableError } from '../util/errorLogFields';
import { modelSsrfFetch } from '../util/outboundFetch';
import type { ILLM, LLMCreateParams, LLMCreateParamsStreaming } from './ILLM';
import {
Expand Down Expand Up @@ -1447,7 +1447,10 @@ export class VercelAILLM implements ILLM {
if (this.signal?.aborted) {
this.logger.debug('LLM call aborted', extractErrorLogFields(streamError));
} else {
this.logger.error('Error creating streaming chat completion', extractErrorLogFields(streamError));
this.logger.error('Error creating streaming chat completion', {
...extractErrorLogFields(streamError),
...(isReportableError(error) ? { reportable_error: { error: streamError } } : {}),
});
}
throw streamError;
}
Expand Down Expand Up @@ -1486,7 +1489,10 @@ export class VercelAILLM implements ILLM {
if (this.signal?.aborted) {
this.logger.debug('LLM stream aborted', extractErrorLogFields(streamError));
} else {
this.logger.error('Error reading LLM stream', extractErrorLogFields(streamError));
this.logger.error('Error reading LLM stream', {
...extractErrorLogFields(streamError),
...(isReportableError(error) ? { reportable_error: { error: streamError } } : {}),
});
Comment thread
ramantehlan marked this conversation as resolved.
}
throw streamError;
}
Expand Down
10 changes: 7 additions & 3 deletions packages/trueforge-core/src/core/runtime/AgentThread.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import type {
} from '../capabilities/AgentContextProcessor';
import { SUB_AGENT_IDENTITY } from '../capabilities/builtins/DynamicSubAgents';
import type { ToolResponseProcessor } from '../capabilities/ToolResponseProcessor';
import { AgentHarnessError, InvalidAgentSendInputError } from '../errors';
import { AgentHarnessError, InvalidAgentSendInputError, ModelProviderError } from '../errors';
import type { RegisteredPassthroughEvent } from '../events/PassthroughEvents';
import {
EventType,
Expand Down Expand Up @@ -55,7 +55,7 @@ import { executeToolCalls } from '../mcp/executeToolCalls';
import type { IToolSet, MCPAuthRequired } from '../mcp/IMCPServer';
import type { HarnessSandbox, SandboxInfo } from '../sandbox/Sandbox';
import type { AgentTracing } from '../tracing/AgentTracing';
import { describeUnknownError, extractErrorLogFields } from '../util/errorLogFields';
import { describeUnknownError, extractErrorLogFields, isReportableError } from '../util/errorLogFields';
import type { AgentDefinition } from './AgentDefinition';
import type {
AgentThreadConstructorInput,
Expand Down Expand Up @@ -1516,7 +1516,11 @@ export class AgentThread {
} catch (error) {
// Providers / transports often reject with plain objects; instanceof Error would
// otherwise collapse those into an opaque "Unknown error occurred" in Agent Steps.
this.logger.error('Agent thread execution failed', extractErrorLogFields(error));
// Model-provider failures are classified at the LLM call. turn.done only keeps the message.
this.logger.error('Agent thread execution failed', {
...extractErrorLogFields(error),
...(error instanceof ModelProviderError || !isReportableError(error) ? {} : { reportable_error: { error } }),
});
yield this.generateErrorEvent(describeUnknownError(error));
} finally {
this.contextBusy = false;
Expand Down
6 changes: 5 additions & 1 deletion packages/trueforge-core/src/core/sandbox/Sandbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,10 @@ export class Sandbox<TEnvironment = undefined> extends LocalToolMCP {
try {
({ sandboxInfo, sandboxCreated } = await this.ensureReadySandbox());
} catch (e) {
this.logger.error('Sandbox initialization failed', extractErrorLogFields(e));
this.logger.error('Sandbox initialization failed', {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
const message = e instanceof Error ? e.message : 'Sandbox initialization failed';
const fallback = this.existingSandboxInfo;
return toolResultResponse({
Expand Down Expand Up @@ -801,6 +804,7 @@ export class Sandbox<TEnvironment = undefined> extends LocalToolMCP {
this.logger.error('Code Mode transport start failed', {
...extractErrorLogFields(e),
sandboxId,
reportable_error: { error: e, extra: { sandboxId } },
});
return undefined;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@ export class CodeModeDispatcher {
} catch (e) {
const source = classifyErrorSource(e);
const errorMessage = e instanceof Error ? e.message : 'Unknown Code Mode dispatch error';
this.logger.error(`Code Mode dispatch failed (source=${source})`, extractErrorLogFields(e));
this.logger.error(`Code Mode dispatch failed (source=${source})`, {
...extractErrorLogFields(e),
...(source === 'internal' ? { reportable_error: { error: e, extra: { source } } } : {}),
});
return { ok: false, error: errorMessage, source };
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,10 @@ export class CodeModeNatsTransport implements CodeModeTransport {
try {
await withTimeout(nc.drain(), NATS_DRAIN_TIMEOUT_MS);
} catch (e) {
this.logger.warn('Code Mode NATS drain failed/timed out; forcing close', extractErrorLogFields(e));
this.logger.warn('Code Mode NATS drain failed/timed out; forcing close', {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
await nc.close().catch(() => {
/* no-op */
});
Expand Down Expand Up @@ -160,7 +163,10 @@ export class CodeModeNatsTransport implements CodeModeTransport {

private async handleMessage(err: Error | null, msg: Msg): Promise<void> {
if (err) {
this.logger.error('Code Mode NATS subscription error', extractErrorLogFields(err));
this.logger.error('Code Mode NATS subscription error', {
...extractErrorLogFields(err),
reportable_error: { error: err },
});
return;
}
if (!msg.reply) {
Expand All @@ -173,7 +179,10 @@ export class CodeModeNatsTransport implements CodeModeTransport {
try {
msg.respond(this.encode({ ok: false, error: 'Code Mode dispatcher is not configured', source: 'internal' }));
} catch (e) {
this.logger.warn(`Failed to send dispatcher-missing reply on '${msg.subject}'`, extractErrorLogFields(e));
this.logger.warn(`Failed to send dispatcher-missing reply on '${msg.subject}'`, {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
}
return;
}
Expand All @@ -192,11 +201,17 @@ export class CodeModeNatsTransport implements CodeModeTransport {
try {
request = this.decode(msg);
} catch (e) {
this.logger.error(`Failed to decode Code Mode NATS request on '${msg.subject}'`, extractErrorLogFields(e));
this.logger.error(`Failed to decode Code Mode NATS request on '${msg.subject}'`, {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
try {
msg.respond(this.encode({ ok: false, error: 'Malformed Code Mode request', source: 'caller' }));
} catch (respondErr) {
this.logger.warn(`Failed to send decode-error reply on '${msg.subject}'`, extractErrorLogFields(respondErr));
this.logger.warn(`Failed to send decode-error reply on '${msg.subject}'`, {
...extractErrorLogFields(respondErr),
reportable_error: { error: respondErr },
});
}
return;
}
Expand All @@ -205,7 +220,10 @@ export class CodeModeNatsTransport implements CodeModeTransport {
try {
msg.respond(this.encode(reply));
} catch (e) {
this.logger.error('Failed to publish Code Mode NATS reply', extractErrorLogFields(e));
this.logger.error('Failed to publish Code Mode NATS reply', {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,6 @@ export interface DaytonaSandboxProviderOptions {
/** Defaults to 1 hour (same as the gateway's max agent execution time). */
previewUrlExpirySeconds?: number;
logger: Logger;
/** Host alert hook (e.g. Sentry P1). */
onCriticalAlert?: (error: Error) => void;
}

export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvironment> {
Expand All @@ -99,7 +97,6 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
private readonly apiKey: string;
private readonly apiUrl: string;
private readonly logger: Logger;
private readonly onCriticalAlert: ((error: Error) => void) | undefined;
private readonly daytona: Daytona;
private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
// De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.
Expand All @@ -117,7 +114,6 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
this.fileMaxBytesForDownload = options.fileMaxBytesForDownload;
this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT;
this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS;
this.onCriticalAlert = options.onCriticalAlert;
this.logger = options.logger.child({ module: 'DaytonaProvider' });
}

Expand Down Expand Up @@ -360,8 +356,11 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
this.logger.error('Daytona sandbox create failed', {
snapshot_ref: env.snapshot_ref,
...extractErrorLogFields(alertError),
reportable_error: {
error: alertError,
extra: { snapshot_ref: env.snapshot_ref },
},
});
this.onCriticalAlert?.(alertError);
throw error;
}
}
Expand Down Expand Up @@ -444,6 +443,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
this.logger.error('Sandbox recovery failed', {
...extractErrorLogFields(recoveryError),
originalError: extractErrorLogFields(originalError),
reportable_error: { error: recoveryError },
});
throw new Error('Sandbox is unavailable; recovery attempt failed.', { cause: recoveryError });
}
Expand All @@ -458,6 +458,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
this.logger.error('Sandbox operation failed after successful recovery', {
...extractErrorLogFields(retryError),
originalError: extractErrorLogFields(originalError),
reportable_error: { error: retryError },
});
// Rethrow as-is so callers can match domain error types (SandboxPathIsDirectoryError, DaytonaError statusCode mappings, etc.).
throw retryError;
Expand Down Expand Up @@ -509,7 +510,10 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
if (e instanceof SandboxNotAvailableError) {
throw e;
}
this.logger.error('Sandbox execution error', extractErrorLogFields(e));
this.logger.error('Sandbox execution error', {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
const message = e instanceof Error ? e.message : 'Unknown error';
return { success: false, error: message };
}
Expand Down Expand Up @@ -575,7 +579,10 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
});
} catch (e: unknown) {
DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId);
this.logger.error('Failed to create signed preview URL', extractErrorLogFields(e));
this.logger.error('Failed to create signed preview URL', {
...extractErrorLogFields(e),
reportable_error: { error: e },
});
throw e;
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,17 @@ export class TFYSandboxProvider implements SandboxProvider {

if (!response.ok) {
const text = await response.text();
this.logger.error(`Sandbox server returned ${String(response.status)}: ${text}`, { url: execUrl });
this.logger.error(`Sandbox server returned ${String(response.status)}: ${text}`, {
url: execUrl,
...(response.status >= 500
? {
reportable_error: {
error: new Error(`Sandbox server returned ${String(response.status)}`),
extra: { status: response.status },
},
}
: {}),
});
return {
success: false,
error: `Sandbox server returned ${String(response.status)} from ${execUrl}: ${text}`,
Expand All @@ -184,13 +194,18 @@ export class TFYSandboxProvider implements SandboxProvider {
this.logger.error(`Sandbox exec timed out after ${String(timeoutSeconds)}s`, {
url: execUrl,
...extractErrorLogFields(e),
reportable_error: { error: e },
});
return {
success: false,
error: `Sandbox exec to ${execUrl} timed out after ${String(timeoutSeconds)}s`,
};
}
this.logger.error('Sandbox exec failed', { url: execUrl, ...extractErrorLogFields(e) });
this.logger.error('Sandbox exec failed', {
url: execUrl,
...extractErrorLogFields(e),
reportable_error: { error: e },
});
return {
success: false,
error: `Sandbox exec to ${execUrl} failed: ${describeUnknownError(e)}`,
Expand Down
34 changes: 34 additions & 0 deletions packages/trueforge-core/src/core/util/errorLogFields.ts
Comment thread
ramantehlan marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,40 @@ function stackWithCauses(error: Error): string | undefined {
return stacks.length > 0 ? stacks.join('\nCaused by: ') : undefined;
}

function readIntegerProperty(error: object, key: string): number | undefined {
if (!(key in error)) {
return undefined;
}
const value: unknown = Reflect.get(error, key);
return typeof value === 'number' && Number.isInteger(value) ? value : undefined;
}

/** `statusCode` (AI SDK, MCP) or `status` (OAuth, HTTPException). */
function readHttpStatus(error: object): number | undefined {
return readIntegerProperty(error, 'statusCode') ?? readIntegerProperty(error, 'status');
}

function httpStatusInChain(error: unknown, seen: Set<unknown>): number | undefined {
if (typeof error !== 'object' || error === null || seen.has(error)) {
return undefined;
}
seen.add(error);
const status = readHttpStatus(error);
if (status !== undefined) {
return status;
}
if (!('cause' in error)) {
return undefined;
}
return httpStatusInChain(Reflect.get(error, 'cause'), seen);
}

/** Every failure is reported except an HTTP status from 200 through 499, including on `cause`. */
export function isReportableError(error: unknown): boolean {
const status = httpStatusInChain(error, new Set());
return status === undefined || status > 499;
}

export function extractErrorLogFields(error: unknown): ErrorLogFields {
if (error instanceof Error) {
const chain = describeErrorChain(error, new Set());
Expand Down
5 changes: 4 additions & 1 deletion packages/trueforge-core/src/core/util/promiseUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ export async function* mergeAsyncGenerators<T>(
}
const nextPromise = generator.next().then(result => ({ idx, result }));
nextPromise.catch((error: unknown) => {
logger.error(`Unexpected error in mergeAsyncGenerators generator ${String(idx)}`, extractErrorLogFields(error));
logger.error(`Unexpected error in mergeAsyncGenerators generator ${String(idx)}`, {
...extractErrorLogFields(error),
reportable_error: { error, extra: { generator: idx } },
});
});
return nextPromise;
};
Expand Down
5 changes: 5 additions & 0 deletions packages/trueforge-core/src/core/util/reportableErrorLog.ts
Comment thread
ramantehlan marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export interface ReportableErrorLog {
error: unknown;
extra?: Record<string, unknown>;
filepath?: string;
}
Loading
Loading