diff --git a/ai-assistants/actions/awsUploadImage.ts b/ai-assistants/actions/awsUploadImage.ts index 34eb20426..4af5523c4 100644 --- a/ai-assistants/actions/awsUploadImage.ts +++ b/ai-assistants/actions/awsUploadImage.ts @@ -1,13 +1,12 @@ import base64ToBlob from "../utils/blobConversion.ts"; import { AssistantIds } from "../types.ts"; import { AppContext } from "../mod.ts"; -import { logger, meter, ValueType } from "@deco/deco/o11y"; -const stats = { - awsUploadImageError: meter.createCounter("assistant_aws_upload_error", { - unit: "1", - valueType: ValueType.INT, - }), -}; +import { logger } from "@deco/deco/o11y"; +import { + ATTR_ASSISTANT_ID, + ATTR_ASSISTANT_OPERATION, + stats, +} from "../observability.ts"; export interface AWSUploadImageProps { file: string | ArrayBuffer | null; assistantIds?: AssistantIds; @@ -49,8 +48,9 @@ export default async function awsUploadImage( const uploadURL = await getSignedUrl(blobData.type, ctx); const uploadResponse = await uploadFileToS3(uploadURL, blobData); if (!uploadResponse.ok) { - stats.awsUploadImageError.add(1, { - assistantId, + stats.errors.add(1, { + [ATTR_ASSISTANT_OPERATION]: "aws_upload", + [ATTR_ASSISTANT_ID]: assistantId, }); throw new Error(`Failed to upload file: ${uploadResponse.statusText}`); } diff --git a/ai-assistants/actions/describeImage.ts b/ai-assistants/actions/describeImage.ts index c5661f712..828491d88 100644 --- a/ai-assistants/actions/describeImage.ts +++ b/ai-assistants/actions/describeImage.ts @@ -1,23 +1,17 @@ import { AssistantIds } from "../types.ts"; import { AppContext } from "../mod.ts"; -import { logger, meter, ValueType } from "@deco/deco/o11y"; +import { logger } from "@deco/deco/o11y"; import { shortcircuit } from "@deco/deco"; - -const stats = { - promptTokens: meter.createHistogram("assistant_image_prompt_tokens", { - description: "Tokens used in Sales Assistant Describe Image Input - OpenAI", - valueType: ValueType.INT, - }), - completionTokens: meter.createHistogram("assistant_image_completion_tokens", { - description: - "Tokens used in Sales Assistant Describe Image Output - OpenAI", - valueType: ValueType.INT, - }), - describeImageError: meter.createCounter("assistant_describe_image_error", { - unit: "1", - valueType: ValueType.INT, - }), -}; +import { + ATTR_ASSISTANT_ID, + ATTR_ASSISTANT_OPERATION, + GEN_AI_SYSTEM, + GEN_AI_SYSTEM_OPENAI, + GEN_AI_TOKEN_TYPE, + GEN_AI_TOKEN_TYPE_INPUT, + GEN_AI_TOKEN_TYPE_OUTPUT, + stats, +} from "../observability.ts"; export interface DescribeImageProps { uploadURL: string; userPrompt: string; @@ -71,11 +65,15 @@ export default async function describeImage( response: JSON.stringify(response), props: describeImageProps, }); - stats.promptTokens.record(response.usage?.prompt_tokens ?? 0, { - assistant_id: assistantId, + stats.tokenUsage.record(response.usage?.prompt_tokens ?? 0, { + [GEN_AI_SYSTEM]: GEN_AI_SYSTEM_OPENAI, + [GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_INPUT, + [ATTR_ASSISTANT_ID]: assistantId, }); - stats.completionTokens.record(response.usage?.completion_tokens ?? 0, { - assistant_id: assistantId, + stats.tokenUsage.record(response.usage?.completion_tokens ?? 0, { + [GEN_AI_SYSTEM]: GEN_AI_SYSTEM_OPENAI, + [GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_OUTPUT, + [ATTR_ASSISTANT_ID]: assistantId, }); return response; } catch (error) { @@ -84,8 +82,9 @@ export default async function describeImage( status: number; headers: Headers; }; - stats.describeImageError.add(1, { - assistantId, + stats.errors.add(1, { + [ATTR_ASSISTANT_OPERATION]: "describe_image", + [ATTR_ASSISTANT_ID]: assistantId, }); shortcircuit( new Response(JSON.stringify({ error: errorObj.error.message }), { diff --git a/ai-assistants/actions/transcribeAudio.ts b/ai-assistants/actions/transcribeAudio.ts index 453417b22..9d5c6d5e6 100644 --- a/ai-assistants/actions/transcribeAudio.ts +++ b/ai-assistants/actions/transcribeAudio.ts @@ -1,22 +1,12 @@ import base64ToBlob from "../utils/blobConversion.ts"; import { AssistantIds } from "../types.ts"; import { AppContext } from "../mod.ts"; -import { logger, meter, ValueType } from "@deco/deco/o11y"; -const stats = { - audioSize: meter.createHistogram("assistant_transcribe_audio_size", { - description: - "Audio size used in Sales Assistant Transcribe Image Input - OpenAI", - unit: "s", - valueType: ValueType.DOUBLE, - }), - transcribeAudioError: meter.createCounter( - "assistant_transcribe_audio_error", - { - unit: "1", - valueType: ValueType.INT, - }, - ), -}; +import { logger } from "@deco/deco/o11y"; +import { + ATTR_ASSISTANT_ID, + ATTR_ASSISTANT_OPERATION, + stats, +} from "../observability.ts"; export interface TranscribeAudioProps { file: string | ArrayBuffer | null; assistantIds?: AssistantIds; @@ -31,8 +21,9 @@ export default async function transcribeAudio( const assistantId = transcribeAudioProps.assistantIds?.assistantId; const threadId = transcribeAudioProps.assistantIds?.threadId; if (!transcribeAudioProps.file) { - stats.transcribeAudioError.add(1, { - assistantId, + stats.errors.add(1, { + [ATTR_ASSISTANT_OPERATION]: "transcribe_audio", + [ATTR_ASSISTANT_ID]: assistantId, }); throw new Error("Audio file is empty"); } @@ -42,8 +33,8 @@ export default async function transcribeAudio( transcribeAudioProps.assistantIds, ); const file = new File([blobData], "input.wav", { type: "audio/wav" }); - stats.audioSize.record(transcribeAudioProps.audioDuration, { - assistant_id: assistantId, + stats.audioDuration.record(transcribeAudioProps.audioDuration, { + [ATTR_ASSISTANT_ID]: assistantId, }); const response = await ctx.openAI.audio.transcriptions.create({ model: "whisper-1", diff --git a/ai-assistants/chat/messages.ts b/ai-assistants/chat/messages.ts index 169eab460..95ee0344d 100644 --- a/ai-assistants/chat/messages.ts +++ b/ai-assistants/chat/messages.ts @@ -1,6 +1,12 @@ import { Context, type JSONSchema7, lazySchemaFor } from "@deco/deco"; -import { meter, ValueType } from "@deco/deco/o11y"; import { weakcache } from "../../utils/weakcache.ts"; +import { + ATTR_ASSISTANT_ID, + ATTR_ASSISTANT_PHASE, + GEN_AI_SYSTEM, + GEN_AI_SYSTEM_OPENAI, + stats, +} from "../observability.ts"; import { ChatMessage, FunctionCallReply, @@ -15,14 +21,6 @@ import { import { threadMessageToReply, Tokens } from "../loaders/messages.ts"; import { AIAssistant, AppContext } from "../mod.ts"; import { dereferenceJsonSchema } from "../schema.ts"; -const stats = { - latency: meter.createHistogram("assistant_latency", { - description: - "assistant latency (time it takes from the moment the server receives the request to the moment it sends the response)", - unit: "ms", - valueType: ValueType.DOUBLE, - }), -}; // Max length of instructions. The maximum context of the assistant is 32K chars. We use 25K for instructions to be safe. const MAX_INSTRUCTIONS_LENGTH = 25000; const notUndefined = (v: T | undefined): v is T => v !== undefined; @@ -224,9 +222,10 @@ export const messageProcessorFor = async ( props, }, }); - stats.latency.record(performance.now() - start, { - type: "start_function_call", - assistant_id: run.assistant_id, + stats.operationDuration.record((performance.now() - start) / 1000, { + [GEN_AI_SYSTEM]: GEN_AI_SYSTEM_OPENAI, + [ATTR_ASSISTANT_PHASE]: "start_function_call", + [ATTR_ASSISTANT_ID]: run.assistant_id, }); }, (call, props, response) => { functionCallReplies.push({ @@ -298,9 +297,10 @@ export const messageProcessorFor = async ( reply(message); } else { reply(replyMessage); - stats.latency.record(performance.now() - start, { - type: "text", - assistant_id: run.assistant_id, + stats.operationDuration.record((performance.now() - start) / 1000, { + [GEN_AI_SYSTEM]: GEN_AI_SYSTEM_OPENAI, + [ATTR_ASSISTANT_PHASE]: "text", + [ATTR_ASSISTANT_ID]: run.assistant_id, }); } if (functionCallReplies.length > 0) { @@ -310,9 +310,10 @@ export const messageProcessorFor = async ( type: "function_calls" as const, content: functionCallReplies, }); - stats.latency.record(performance.now() - start, { - type: "function_calls", - assistant_id: run.assistant_id, + stats.operationDuration.record((performance.now() - start) / 1000, { + [GEN_AI_SYSTEM]: GEN_AI_SYSTEM_OPENAI, + [ATTR_ASSISTANT_PHASE]: "function_calls", + [ATTR_ASSISTANT_ID]: run.assistant_id, }); } }; diff --git a/ai-assistants/observability.ts b/ai-assistants/observability.ts new file mode 100644 index 000000000..5a248152c --- /dev/null +++ b/ai-assistants/observability.ts @@ -0,0 +1,56 @@ +// Shared OTel instruments for the AI assistants app. Standard GenAI telemetry +// uses the official @opentelemetry/semantic-conventions (gen_ai.*) constants; +// deco-proprietary dimensions use the deco.assistant.* namespace. The meter is +// reused from the deco framework. +import { meter, ValueType } from "@deco/deco/o11y"; +import { + ATTR_GEN_AI_SYSTEM, + ATTR_GEN_AI_TOKEN_TYPE, + METRIC_GEN_AI_CLIENT_OPERATION_DURATION, + METRIC_GEN_AI_CLIENT_TOKEN_USAGE, +} from "npm:@opentelemetry/semantic-conventions@1.37.0/incubating"; + +// semconv attribute keys + values +export const GEN_AI_SYSTEM = ATTR_GEN_AI_SYSTEM; +export const GEN_AI_SYSTEM_OPENAI = "openai"; +export const GEN_AI_TOKEN_TYPE = ATTR_GEN_AI_TOKEN_TYPE; +export const GEN_AI_TOKEN_TYPE_INPUT = "input"; +export const GEN_AI_TOKEN_TYPE_OUTPUT = "output"; + +// deco-proprietary attributes (no semconv equivalent) +export const ATTR_ASSISTANT_ID = "assistant_id"; +export const ATTR_ASSISTANT_PHASE = "deco.assistant.phase"; +export const ATTR_ASSISTANT_OPERATION = "deco.assistant.operation"; + +export const stats = { + // gen_ai.client.operation.duration — seconds (semconv) + operationDuration: meter.createHistogram( + METRIC_GEN_AI_CLIENT_OPERATION_DURATION, + { + description: "GenAI assistant operation duration.", + unit: "s", + valueType: ValueType.DOUBLE, + }, + ), + // gen_ai.client.token.usage — split by gen_ai.token.type (input/output) + tokenUsage: meter.createHistogram(METRIC_GEN_AI_CLIENT_TOKEN_USAGE, { + description: "Number of tokens used in GenAI assistant requests.", + unit: "{token}", + valueType: ValueType.INT, + }), + // deco-proprietary: transcribed audio duration (seconds) + audioDuration: meter.createHistogram( + "deco.assistant.transcribe.audio_duration", + { + description: "Duration of audio transcribed by the assistant.", + unit: "s", + valueType: ValueType.DOUBLE, + }, + ), + // deco-proprietary: assistant operation errors, dimensioned by operation + errors: meter.createCounter("deco.assistant.errors", { + description: "Assistant operation errors.", + unit: "1", + valueType: ValueType.INT, + }), +};