diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..fc992155 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +gen/go/agentevals/** linguist-generated=true +gen/go/go.sum linguist-generated=true +src/agentevals/api/v1alpha1/** linguist-generated=true +ui/src/generated/** linguist-generated=true +proto/buf.lock linguist-generated=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43889a71..64d9e2ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,41 @@ jobs: - name: Build UI run: make build-ui + - name: Lint UI + working-directory: ui + run: npm run lint + + proto: + name: Protobuf Contract Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: gen/go/go.mod + cache-dependency-path: gen/go/go.sum + + # proto-breaking compares against the local main branch. + - name: Fetch main + run: git fetch --no-tags --depth=1 origin main:main || true + + - name: Lint, regenerate and check for drift + run: make proto-check + + - name: Breaking changes + run: make proto-breaking + + - name: Build the Go module + working-directory: gen/go + run: | + go build ./... + go vet ./... + docker: runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6fec3fdf..41263cf1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,6 +93,33 @@ jobs: files: dist/**/*.whl generate_release_notes: true + # Go resolves github.com/agentevals-dev/agentevals/gen/go@vX.Y.Z from the tag gen/go/vX.Y.Z. + tag-go-module: + needs: github-release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.inputs.tag || github.ref_name }} + fetch-depth: 0 + + - name: Tag the generated Go module + run: | + if ! [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + echo "Not a semantic version tag: $TAG" + exit 1 + fi + if git rev-parse -q --verify "refs/tags/gen/go/$TAG" >/dev/null; then + echo "gen/go/$TAG already exists" + exit 0 + fi + git tag "gen/go/$TAG" "$TAG^{commit}" + git push origin "gen/go/$TAG" + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + push-docker: needs: github-release runs-on: ubuntu-latest diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 89788a54..bfb18043 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -65,6 +65,19 @@ make build-ui # build React app only → ui/dist/ Both `build` and `build-bundle` produce `dist/agentevals-*.whl` with the same package name and version. The difference is that `build-bundle` embeds `ui/dist/` as `agentevals/_static/` inside the wheel. The hatchling `artifacts` config ensures the gitignored `_static/` directory is included. +### API contract + +The gRPC API is defined in `proto/`. The generated Go, Python and TypeScript code is committed, so only changes to `.proto` files need these: + +```bash +make proto-lint # buf lint +make proto-generate # regenerate gen/go, src/agentevals/api/v1alpha1 and ui/src/generated +make proto-check # lint, regenerate and fail when committed code is out of date (CI) +make proto-breaking # compare with main for breaking changes (CI) +``` + +`buf` runs through `npx`, so Node is enough. Building the Go module (`cd gen/go && go build ./...`) needs Go, which the Nix dev shell provides. + ### Testing ```bash @@ -151,6 +164,8 @@ To release a new Nix derivation, update `flake.nix` with the new version and reb ``` 4. Alternatively, trigger manually from **GitHub → Actions → Release → Run workflow** and enter the tag +The workflow also tags the same commit `gen/go/vX.Y.Z`, the version Go clients get with `go get github.com/agentevals-dev/agentevals/gen/go@vX.Y.Z`. + The workflow (`.github/workflows/release.yml`) runs `make release`, which builds the wheel twice (once without UI, once with embedded UI) into separate subdirectories: ``` diff --git a/Dockerfile b/Dockerfile index b909bf42..78a090fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,6 @@ USER app ENV PATH="/app/.venv/bin:$PATH" ENV AGENTEVALS_SERVER_URL=http://127.0.0.1:8001 -EXPOSE 8001 4318 4317 8080 +EXPOSE 8001 4318 4317 8002 8080 CMD ["agentevals", "serve", "--host", "0.0.0.0", "--port", "8001", "--otlp-http-port", "4318", "--otlp-grpc-port", "4317", "--mcp-port", "8080"] diff --git a/Makefile b/Makefile index 88cd31a4..0879772d 100644 --- a/Makefile +++ b/Makefile @@ -143,3 +143,41 @@ helm-package: helm-cleanup .PHONY: helm-publish helm-publish: helm-package helm push "$(HELM_DIST_FOLDER)/agentevals-$(HELM_CHART_VERSION).tgz" "$(HELM_CHART_OCI_URL)" + +# Protobuf contract. buf runs through npx so no Go toolchain is needed; regenerating spends +# Buf Schema Registry remote plugin requests, which are rate limited. +BUF ?= npx --yes @bufbuild/buf@1.72.0 +PROTO_BREAKING_BRANCH ?= main +PROTO_GENERATED_PATHS := gen/go/agentevals src/agentevals/api/v1alpha1 ui/src/generated + +.PHONY: proto-lint +proto-lint: + cd proto && $(BUF) lint + +.PHONY: proto-generate +proto-generate: + rm -rf $(PROTO_GENERATED_PATHS) + cd proto && $(BUF) generate + +.PHONY: proto-breaking +proto-breaking: + @if git cat-file -e "$(PROTO_BREAKING_BRANCH):proto/buf.yaml" 2>/dev/null; then \ + $(BUF) breaking proto --against ".git#branch=$(PROTO_BREAKING_BRANCH),subdir=proto"; \ + else \ + echo "No protobuf module on $(PROTO_BREAKING_BRANCH); skipping the breaking check"; \ + fi + +.PHONY: proto-check +proto-check: proto-lint proto-generate + @if test -n "$$(git status --porcelain -- $(PROTO_GENERATED_PATHS))"; then \ + echo "Generated protobuf files are out of date:"; \ + git status --short -- $(PROTO_GENERATED_PATHS); \ + exit 1; \ + fi + @# git status skips ignored files, so a .gitignore rule could keep a generated file out of the repo unnoticed. + @ignored="$$(git ls-files --others --ignored --exclude-standard -- $(PROTO_GENERATED_PATHS) | grep -v /__pycache__/)"; \ + if test -n "$$ignored"; then \ + echo "Generated protobuf files are ignored by a .gitignore rule:"; \ + echo "$$ignored"; \ + exit 1; \ + fi diff --git a/README.md b/README.md index 706f22c1..f5313b98 100644 --- a/README.md +++ b/README.md @@ -175,18 +175,21 @@ evaluators: **Web UI.** `agentevals serve` on `http://localhost:8001`. Upload traces and eval sets, pick evaluators, inspect span trees, watch live sessions. API docs at `/docs`. +**gRPC API.** `agentevals serve` also serves a gRPC API on port 8002 for programs: sessions, evaluation, runs and trace conversion, with generated clients for Go and Python. See [gRPC API](docs/grpc-api.md). + **MCP server.** `agentevals mcp` exposes `list_metrics`, `evaluate_traces`, `list_sessions`, `summarize_session` and `evaluate_sessions` to MCP clients. A `.mcp.json` at the repo root lets Claude Code pick it up, and the `/eval` and `/inspect` skills in `.claude/skills/` build on it. **Docker.** ```bash docker build -t agentevals . -docker run -p 8001:8001 -p 4317:4317 -p 4318:4318 agentevals +docker run -p 8001:8001 -p 8002:8002 -p 4317:4317 -p 4318:4318 agentevals ``` | Port | Purpose | |------|---------| | 8001 | Web UI and REST API | +| 8002 | gRPC API | | 4317 | OTLP gRPC receiver (traces and logs) | | 4318 | OTLP HTTP receiver (traces and logs) | | 8080 | MCP (Streamable HTTP) | @@ -221,6 +224,7 @@ The chart runs in memory by default. `--set storage.backend=postgres` with a bun | [Eval Set Format](docs/eval-set-format.md) | Schema and examples for golden eval sets | | [Custom Evaluators](docs/custom-evaluators.md) | Write scoring logic in any language | | [Run History](docs/run-history.md) | Persist evaluations to Postgres and trend them in the UI | +| [gRPC API](docs/grpc-api.md) | Services, generated clients, configuration and security of the gRPC API | ## Development diff --git a/charts/agentevals/templates/NOTES.txt b/charts/agentevals/templates/NOTES.txt index 575abcb7..76fab54f 100644 --- a/charts/agentevals/templates/NOTES.txt +++ b/charts/agentevals/templates/NOTES.txt @@ -2,8 +2,11 @@ 2. OTLP HTTP receiver: port {{ .Values.service.otlpHttp.port }} (OTEL_EXPORTER_OTLP_ENDPOINT=http://:{{ .Values.service.otlpHttp.port }}). 3. OTLP gRPC receiver: port {{ .Values.service.otlpGrpc.port }} (OTEL_EXPORTER_OTLP_ENDPOINT=:{{ .Values.service.otlpGrpc.port }}, OTEL_EXPORTER_OTLP_PROTOCOL=grpc). 4. MCP (Streamable HTTP): port {{ .Values.service.mcp.port }}, path /mcp (e.g. http://:{{ .Values.service.mcp.port }}/mcp). +{{- if .Values.grpcApi.enabled }} +5. gRPC API (agentevals.api.v1alpha1): port {{ .Values.service.grpcApi.port }} (Service port name: grpc-api). Expose it only to trusted callers. +{{- end }} {{- if .Values.ephemeralVolume.enabled }} -5. An emptyDir is mounted at /tmp with HOME=/tmp/agentevals-home (ephemeral; lost on pod restart). Set ephemeralVolume.enabled=false and readOnlyRootFilesystem=false if you need a writable root without this mount. +{{ ternary 6 5 .Values.grpcApi.enabled }}. An emptyDir is mounted at /tmp with HOME=/tmp/agentevals-home (ephemeral; lost on pod restart). Set ephemeralVolume.enabled=false and readOnlyRootFilesystem=false if you need a writable root without this mount. {{- end }} Get the Service URL: diff --git a/charts/agentevals/templates/deployment.yaml b/charts/agentevals/templates/deployment.yaml index 2fe9780e..6dfc873a 100644 --- a/charts/agentevals/templates/deployment.yaml +++ b/charts/agentevals/templates/deployment.yaml @@ -64,6 +64,12 @@ spec: env: - name: AGENTEVALS_SERVER_URL value: "http://127.0.0.1:{{ .Values.service.http.containerPort }}" + - name: AGENTEVALS_API_GRPC_PORT + value: {{ ternary .Values.service.grpcApi.containerPort 0 .Values.grpcApi.enabled | quote }} + {{- if and .Values.grpcApi.enabled .Values.grpcApi.reflection }} + - name: AGENTEVALS_GRPC_REFLECTION + value: "true" + {{- end }} {{- if .Values.ephemeralVolume.enabled }} - name: TMPDIR value: "/tmp" @@ -117,6 +123,11 @@ spec: - name: mcp containerPort: {{ .Values.service.mcp.containerPort }} protocol: TCP + {{- if .Values.grpcApi.enabled }} + - name: grpc-api + containerPort: {{ .Values.service.grpcApi.containerPort }} + protocol: TCP + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} securityContext: @@ -133,9 +144,14 @@ spec: periodSeconds: 10 timeoutSeconds: 5 readinessProbe: + {{- if and .Values.grpcApi.enabled .Values.grpcApi.grpcReadinessProbe }} + grpc: + port: {{ .Values.service.grpcApi.containerPort }} + {{- else }} httpGet: path: /api/health port: http + {{- end }} initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: diff --git a/charts/agentevals/templates/service.yaml b/charts/agentevals/templates/service.yaml index f224c08c..f724f4a7 100644 --- a/charts/agentevals/templates/service.yaml +++ b/charts/agentevals/templates/service.yaml @@ -24,5 +24,12 @@ spec: port: {{ .Values.service.mcp.port }} targetPort: mcp protocol: TCP + {{- if .Values.grpcApi.enabled }} + - name: grpc-api + port: {{ .Values.service.grpcApi.port }} + targetPort: grpc-api + protocol: TCP + appProtocol: grpc + {{- end }} selector: {{- include "agentevals.app.selectorLabels" . | nindent 4 }} diff --git a/charts/agentevals/values.yaml b/charts/agentevals/values.yaml index 50e5c660..af95b359 100644 --- a/charts/agentevals/values.yaml +++ b/charts/agentevals/values.yaml @@ -141,6 +141,24 @@ service: mcp: port: 8080 containerPort: 8080 + # -- gRPC API port (agentevals.api.v1alpha1) + grpcApi: + port: 8002 + containerPort: 8002 + +# ============================================================================== +# gRPC API +# ============================================================================== + +grpcApi: + # -- Serve the gRPC API next to the HTTP API. Every caller that can reach the port is + # trusted, so expose it only to trusted callers. + enabled: true + # -- Turn on gRPC server reflection, for tools such as grpcurl + reflection: false + # -- Check readiness with the gRPC health service on the gRPC API port instead of HTTP + # /api/health + grpcReadinessProbe: false # ============================================================================== # Resources diff --git a/docs/grpc-api.md b/docs/grpc-api.md new file mode 100644 index 00000000..7b6ae6cd --- /dev/null +++ b/docs/grpc-api.md @@ -0,0 +1,71 @@ +# gRPC API + +`agentevals serve` serves a gRPC API on port 8002, next to the web UI and REST API on 8001. It is the API for programs: other services, scripts and CI. It speaks native gRPC over HTTP/2, so browsers cannot call it; the web UI uses the REST API and an SSE stream that carry the same messages as JSON. + +The contract lives in [`proto/agentevals/api/v1alpha1`](../proto/agentevals/api/v1alpha1). `v1alpha1` can still change; CI checks every change for compatibility. + +## Services + +| Service | Methods | +|---------|---------| +| `SessionService` | `ListSessions`, `GetSession`, `WatchSessions` (every current session, then each change) | +| `EvaluationService` | `ListMetrics`, `EvaluateTraces` (streams progress and results) | +| `RunService` | `SubmitRun`, `GetRun`, `ListRuns`, `CancelRun`, `ListRunResults`, `WatchRun`. Needs the Postgres backend, see [Run History](run-history.md) | +| `ConversionService` | `ConvertTraces` (trace files to conversations and a golden eval set) | +| `SystemService` | `GetVersion`, `GetConfig` | + +The server also implements the standard `grpc.health.v1.Health` service, so Kubernetes `grpc` probes and load balancers can check it. + +Sessions and conversations come back as views: turns with their user text, tool calls paired with their results, model calls, token usage and status, already worked out by the server. Each field names the OpenTelemetry attribute it comes from. A token count that was not recorded is left unset, never reported as 0. + +Traces go in as OTLP protobuf (`TracesData` and `LogsData`) or as files (OTLP/JSON, OTLP/JSONL or Jaeger JSON), the same formats the REST API takes. + +## Calling it + +With reflection on, `grpcurl` can list and call everything: + +```bash +AGENTEVALS_GRPC_REFLECTION=true uv run agentevals serve --dev +grpcurl -plaintext localhost:8002 list +grpcurl -plaintext localhost:8002 agentevals.api.v1alpha1.SessionService/ListSessions +grpcurl -plaintext -d '{"includeTurns": true}' localhost:8002 agentevals.api.v1alpha1.SessionService/WatchSessions +``` + +Reflection is off by default. Without it, pass the protos with `-import-path` and `-proto`. + +Generated clients: + +* **Go:** `go get github.com/agentevals-dev/agentevals/gen/go@vX.Y.Z`, then import `github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1`. +* **Python:** the `agentevals-cli` package ships the stubs as `agentevals.api.v1alpha1`. +* **TypeScript:** generate from the protos with `buf` and `protoc-gen-es`, as `ui/src/generated` does. + +Errors use standard gRPC status codes with a `google.rpc.ErrorInfo` detail whose `reason` is stable, for example `STORAGE_NOT_CONFIGURED` or `RUN_SPEC_CONFLICT`. Internal errors return `INTERNAL` with no details; the server log has them. + +## Configuration + +| Setting | Purpose | +|---------|---------| +| `--api-grpc-port` or `AGENTEVALS_API_GRPC_PORT` | Port, default `8002`; `0` turns the API off | +| `AGENTEVALS_GRPC_REFLECTION=true` | Turn on server reflection | +| `AGENTEVALS_API_TRACES=true` | Export traces of the API server itself (see below) | + +Messages can be up to 16 MiB. Uploaded trace documents are limited to 10 MiB each, as in the REST API. + +The Helm chart has the same switches under `grpcApi`: `enabled`, `reflection`, and `grpcReadinessProbe` to check readiness with the gRPC health service. The Service exposes the port as `grpc-api`. + +## Tracing the API server + +agentevals receives telemetry, so it exports none of its own by default. With `AGENTEVALS_API_TRACES=true` it traces every API call with the OpenTelemetry gRPC instrumentation, using the standard exporter variables: + +```bash +AGENTEVALS_API_TRACES=true OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318 uv run agentevals serve +AGENTEVALS_API_TRACES=true OTEL_TRACES_EXPORTER=console uv run agentevals serve +``` + +`OTEL_SDK_DISABLED=true` or `OTEL_TRACES_EXPORTER=none` turn it off again. Health checks and reflection are not traced. If these traces are sent to agentevals itself, it drops them, so they never show up as sessions. + +## Security + +The API has no authentication. Any caller that can reach the port can read sessions, run evaluations with the server's credentials, and submit runs. Expose port 8002 only to trusted callers, for example inside a cluster behind a NetworkPolicy, and put authentication in front of it when you need it. + +On a laptop `agentevals serve` listens on all interfaces. Pass `--host 127.0.0.1` to keep every port local. diff --git a/flake.nix b/flake.nix index 50ef4eb2..5fc9cf69 100644 --- a/flake.nix +++ b/flake.nix @@ -90,6 +90,9 @@ # NodeJS pkgs.nodejs_22 + # Go, to build and tidy the generated API module in gen/go + pkgs.go + # C++ standard library for numpy pkgs.stdenv.cc.cc.lib diff --git a/gen/go/README.md b/gen/go/README.md new file mode 100644 index 00000000..d5b0be01 --- /dev/null +++ b/gen/go/README.md @@ -0,0 +1,10 @@ +# agentevals Go API client + +Generated Go types and gRPC clients for the `agentevals.api.v1alpha1` API. + +```go +import apiv1alpha1 "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1" +``` + +The code is generated from `proto/` by `make proto-generate` and must not be edited by hand. +Releases are tagged `gen/go/vX.Y.Z`, matching the agentevals version. diff --git a/gen/go/agentevals/api/v1alpha1/common.pb.go b/gen/go/agentevals/api/v1alpha1/common.pb.go new file mode 100644 index 00000000..b1f0d85c --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/common.pb.go @@ -0,0 +1,501 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/common.proto + +package apiv1alpha1 + +import ( + v11 "go.opentelemetry.io/proto/otlp/logs/v1" + v1 "go.opentelemetry.io/proto/otlp/trace/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TraceFormat int32 + +const ( + // Detected from the content. + TraceFormat_TRACE_FORMAT_UNSPECIFIED TraceFormat = 0 + // One OTLP/JSON document (hex ids, as the OTLP/HTTP JSON encoding). + TraceFormat_TRACE_FORMAT_OTLP_JSON TraceFormat = 1 + // One OTLP/JSON document or span per line. + TraceFormat_TRACE_FORMAT_OTLP_JSONL TraceFormat = 2 + // A Jaeger JSON export. + TraceFormat_TRACE_FORMAT_JAEGER_JSON TraceFormat = 3 +) + +// Enum value maps for TraceFormat. +var ( + TraceFormat_name = map[int32]string{ + 0: "TRACE_FORMAT_UNSPECIFIED", + 1: "TRACE_FORMAT_OTLP_JSON", + 2: "TRACE_FORMAT_OTLP_JSONL", + 3: "TRACE_FORMAT_JAEGER_JSON", + } + TraceFormat_value = map[string]int32{ + "TRACE_FORMAT_UNSPECIFIED": 0, + "TRACE_FORMAT_OTLP_JSON": 1, + "TRACE_FORMAT_OTLP_JSONL": 2, + "TRACE_FORMAT_JAEGER_JSON": 3, + } +) + +func (x TraceFormat) Enum() *TraceFormat { + p := new(TraceFormat) + *p = x + return p +} + +func (x TraceFormat) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TraceFormat) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_common_proto_enumTypes[0].Descriptor() +} + +func (TraceFormat) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_common_proto_enumTypes[0] +} + +func (x TraceFormat) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TraceFormat.Descriptor instead. +func (TraceFormat) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{0} +} + +// How traces form evaluation units. +type GroupBy int32 + +const ( + // The default of the field that uses it. + GroupBy_GROUP_BY_UNSPECIFIED GroupBy = 0 + // Conversation when any trace carries agentevals.session_name or the eval set has a multi + // turn case, otherwise trace. + GroupBy_GROUP_BY_AUTO GroupBy = 1 + // Every trace on its own. + GroupBy_GROUP_BY_TRACE GroupBy = 2 + // Traces sharing agentevals.session_name, gen_ai.conversation.id or session.id form one unit. + GroupBy_GROUP_BY_CONVERSATION GroupBy = 3 +) + +// Enum value maps for GroupBy. +var ( + GroupBy_name = map[int32]string{ + 0: "GROUP_BY_UNSPECIFIED", + 1: "GROUP_BY_AUTO", + 2: "GROUP_BY_TRACE", + 3: "GROUP_BY_CONVERSATION", + } + GroupBy_value = map[string]int32{ + "GROUP_BY_UNSPECIFIED": 0, + "GROUP_BY_AUTO": 1, + "GROUP_BY_TRACE": 2, + "GROUP_BY_CONVERSATION": 3, + } +) + +func (x GroupBy) Enum() *GroupBy { + p := new(GroupBy) + *p = x + return p +} + +func (x GroupBy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GroupBy) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_common_proto_enumTypes[1].Descriptor() +} + +func (GroupBy) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_common_proto_enumTypes[1] +} + +func (x GroupBy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GroupBy.Descriptor instead. +func (GroupBy) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{1} +} + +// limit 0 means the default of 50; the maximum is 100. +type PageRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Limit int32 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` + // Opaque token from a previous PageResponse. + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PageRequest) Reset() { + *x = PageRequest{} + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PageRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PageRequest) ProtoMessage() {} + +func (x *PageRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PageRequest.ProtoReflect.Descriptor instead. +func (*PageRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{0} +} + +func (x *PageRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *PageRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +type PageResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Empty on the last page. + NextPageToken string `protobuf:"bytes,1,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PageResponse) Reset() { + *x = PageResponse{} + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PageResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PageResponse) ProtoMessage() {} + +func (x *PageResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PageResponse.ProtoReflect.Descriptor instead. +func (*PageResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{1} +} + +func (x *PageResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +// A trace file as bytes, as the REST API takes uploads. At most 10 MiB. +type TraceDocument struct { + state protoimpl.MessageState `protogen:"open.v1"` + Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + Format TraceFormat `protobuf:"varint,2,opt,name=format,proto3,enum=agentevals.api.v1alpha1.TraceFormat" json:"format,omitempty"` + // The file name, used in messages and as a session name hint (trace_.json). + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TraceDocument) Reset() { + *x = TraceDocument{} + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceDocument) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceDocument) ProtoMessage() {} + +func (x *TraceDocument) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceDocument.ProtoReflect.Descriptor instead. +func (*TraceDocument) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{2} +} + +func (x *TraceDocument) GetContent() []byte { + if x != nil { + return x.Content + } + return nil +} + +func (x *TraceDocument) GetFormat() TraceFormat { + if x != nil { + return x.Format + } + return TraceFormat_TRACE_FORMAT_UNSPECIFIED +} + +func (x *TraceDocument) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// OTLP telemetry as protobuf. Logs carry the GenAI events that belong to the spans. +type OtlpTelemetry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Traces *v1.TracesData `protobuf:"bytes,1,opt,name=traces,proto3" json:"traces,omitempty"` + Logs *v11.LogsData `protobuf:"bytes,2,opt,name=logs,proto3" json:"logs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OtlpTelemetry) Reset() { + *x = OtlpTelemetry{} + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OtlpTelemetry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OtlpTelemetry) ProtoMessage() {} + +func (x *OtlpTelemetry) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OtlpTelemetry.ProtoReflect.Descriptor instead. +func (*OtlpTelemetry) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{3} +} + +func (x *OtlpTelemetry) GetTraces() *v1.TracesData { + if x != nil { + return x.Traces + } + return nil +} + +func (x *OtlpTelemetry) GetLogs() *v11.LogsData { + if x != nil { + return x.Logs + } + return nil +} + +// Traces to read. Both fields may be set; their traces are combined. +type TraceInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Otlp *OtlpTelemetry `protobuf:"bytes,1,opt,name=otlp,proto3" json:"otlp,omitempty"` + Documents []*TraceDocument `protobuf:"bytes,2,rep,name=documents,proto3" json:"documents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TraceInput) Reset() { + *x = TraceInput{} + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceInput) ProtoMessage() {} + +func (x *TraceInput) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_common_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceInput.ProtoReflect.Descriptor instead. +func (*TraceInput) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_common_proto_rawDescGZIP(), []int{4} +} + +func (x *TraceInput) GetOtlp() *OtlpTelemetry { + if x != nil { + return x.Otlp + } + return nil +} + +func (x *TraceInput) GetDocuments() []*TraceDocument { + if x != nil { + return x.Documents + } + return nil +} + +var File_agentevals_api_v1alpha1_common_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_common_proto_rawDesc = "" + + "\n" + + "$agentevals/api/v1alpha1/common.proto\x12\x17agentevals.api.v1alpha1\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\"B\n" + + "\vPageRequest\x12\x14\n" + + "\x05limit\x18\x01 \x01(\x05R\x05limit\x12\x1d\n" + + "\n" + + "page_token\x18\x02 \x01(\tR\tpageToken\"6\n" + + "\fPageResponse\x12&\n" + + "\x0fnext_page_token\x18\x01 \x01(\tR\rnextPageToken\"{\n" + + "\rTraceDocument\x12\x18\n" + + "\acontent\x18\x01 \x01(\fR\acontent\x12<\n" + + "\x06format\x18\x02 \x01(\x0e2$.agentevals.api.v1alpha1.TraceFormatR\x06format\x12\x12\n" + + "\x04name\x18\x03 \x01(\tR\x04name\"\x8c\x01\n" + + "\rOtlpTelemetry\x12@\n" + + "\x06traces\x18\x01 \x01(\v2(.opentelemetry.proto.trace.v1.TracesDataR\x06traces\x129\n" + + "\x04logs\x18\x02 \x01(\v2%.opentelemetry.proto.logs.v1.LogsDataR\x04logs\"\x8e\x01\n" + + "\n" + + "TraceInput\x12:\n" + + "\x04otlp\x18\x01 \x01(\v2&.agentevals.api.v1alpha1.OtlpTelemetryR\x04otlp\x12D\n" + + "\tdocuments\x18\x02 \x03(\v2&.agentevals.api.v1alpha1.TraceDocumentR\tdocuments*\x82\x01\n" + + "\vTraceFormat\x12\x1c\n" + + "\x18TRACE_FORMAT_UNSPECIFIED\x10\x00\x12\x1a\n" + + "\x16TRACE_FORMAT_OTLP_JSON\x10\x01\x12\x1b\n" + + "\x17TRACE_FORMAT_OTLP_JSONL\x10\x02\x12\x1c\n" + + "\x18TRACE_FORMAT_JAEGER_JSON\x10\x03*e\n" + + "\aGroupBy\x12\x18\n" + + "\x14GROUP_BY_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rGROUP_BY_AUTO\x10\x01\x12\x12\n" + + "\x0eGROUP_BY_TRACE\x10\x02\x12\x19\n" + + "\x15GROUP_BY_CONVERSATION\x10\x03BQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_common_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_common_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_common_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_common_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_common_proto_rawDesc), len(file_agentevals_api_v1alpha1_common_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_common_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_common_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_agentevals_api_v1alpha1_common_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_agentevals_api_v1alpha1_common_proto_goTypes = []any{ + (TraceFormat)(0), // 0: agentevals.api.v1alpha1.TraceFormat + (GroupBy)(0), // 1: agentevals.api.v1alpha1.GroupBy + (*PageRequest)(nil), // 2: agentevals.api.v1alpha1.PageRequest + (*PageResponse)(nil), // 3: agentevals.api.v1alpha1.PageResponse + (*TraceDocument)(nil), // 4: agentevals.api.v1alpha1.TraceDocument + (*OtlpTelemetry)(nil), // 5: agentevals.api.v1alpha1.OtlpTelemetry + (*TraceInput)(nil), // 6: agentevals.api.v1alpha1.TraceInput + (*v1.TracesData)(nil), // 7: opentelemetry.proto.trace.v1.TracesData + (*v11.LogsData)(nil), // 8: opentelemetry.proto.logs.v1.LogsData +} +var file_agentevals_api_v1alpha1_common_proto_depIdxs = []int32{ + 0, // 0: agentevals.api.v1alpha1.TraceDocument.format:type_name -> agentevals.api.v1alpha1.TraceFormat + 7, // 1: agentevals.api.v1alpha1.OtlpTelemetry.traces:type_name -> opentelemetry.proto.trace.v1.TracesData + 8, // 2: agentevals.api.v1alpha1.OtlpTelemetry.logs:type_name -> opentelemetry.proto.logs.v1.LogsData + 5, // 3: agentevals.api.v1alpha1.TraceInput.otlp:type_name -> agentevals.api.v1alpha1.OtlpTelemetry + 4, // 4: agentevals.api.v1alpha1.TraceInput.documents:type_name -> agentevals.api.v1alpha1.TraceDocument + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_common_proto_init() } +func file_agentevals_api_v1alpha1_common_proto_init() { + if File_agentevals_api_v1alpha1_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_common_proto_rawDesc), len(file_agentevals_api_v1alpha1_common_proto_rawDesc)), + NumEnums: 2, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_agentevals_api_v1alpha1_common_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_common_proto_depIdxs, + EnumInfos: file_agentevals_api_v1alpha1_common_proto_enumTypes, + MessageInfos: file_agentevals_api_v1alpha1_common_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_common_proto = out.File + file_agentevals_api_v1alpha1_common_proto_goTypes = nil + file_agentevals_api_v1alpha1_common_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/conversion.pb.go b/gen/go/agentevals/api/v1alpha1/conversion.pb.go new file mode 100644 index 00000000..2b2a6401 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/conversion.pb.go @@ -0,0 +1,263 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/conversion.proto + +package apiv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ConvertTracesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Traces *TraceInput `protobuf:"bytes,1,opt,name=traces,proto3" json:"traces,omitempty"` + // Unset means GROUP_BY_TRACE. + GroupBy GroupBy `protobuf:"varint,2,opt,name=group_by,json=groupBy,proto3,enum=agentevals.api.v1alpha1.GroupBy" json:"group_by,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConvertTracesRequest) Reset() { + *x = ConvertTracesRequest{} + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConvertTracesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConvertTracesRequest) ProtoMessage() {} + +func (x *ConvertTracesRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConvertTracesRequest.ProtoReflect.Descriptor instead. +func (*ConvertTracesRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_conversion_proto_rawDescGZIP(), []int{0} +} + +func (x *ConvertTracesRequest) GetTraces() *TraceInput { + if x != nil { + return x.Traces + } + return nil +} + +func (x *ConvertTracesRequest) GetGroupBy() GroupBy { + if x != nil { + return x.GroupBy + } + return GroupBy_GROUP_BY_UNSPECIFIED +} + +type ConvertedConversation struct { + state protoimpl.MessageState `protogen:"open.v1"` + Conversation *ConversationView `protobuf:"bytes,1,opt,name=conversation,proto3" json:"conversation,omitempty"` + // The conversation as an ADK EvalSet with one case, to use as a golden. Without captured + // content (see TurnView.content_captured) its user turns are empty. + GoldenAdkEvalSet *structpb.Struct `protobuf:"bytes,2,opt,name=golden_adk_eval_set,json=goldenAdkEvalSet,proto3" json:"golden_adk_eval_set,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConvertedConversation) Reset() { + *x = ConvertedConversation{} + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConvertedConversation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConvertedConversation) ProtoMessage() {} + +func (x *ConvertedConversation) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConvertedConversation.ProtoReflect.Descriptor instead. +func (*ConvertedConversation) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_conversion_proto_rawDescGZIP(), []int{1} +} + +func (x *ConvertedConversation) GetConversation() *ConversationView { + if x != nil { + return x.Conversation + } + return nil +} + +func (x *ConvertedConversation) GetGoldenAdkEvalSet() *structpb.Struct { + if x != nil { + return x.GoldenAdkEvalSet + } + return nil +} + +type ConvertTracesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Conversations []*ConvertedConversation `protobuf:"bytes,1,rep,name=conversations,proto3" json:"conversations,omitempty"` + // Documents that could not be read. + Warnings []string `protobuf:"bytes,2,rep,name=warnings,proto3" json:"warnings,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConvertTracesResponse) Reset() { + *x = ConvertTracesResponse{} + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConvertTracesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConvertTracesResponse) ProtoMessage() {} + +func (x *ConvertTracesResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_conversion_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConvertTracesResponse.ProtoReflect.Descriptor instead. +func (*ConvertTracesResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_conversion_proto_rawDescGZIP(), []int{2} +} + +func (x *ConvertTracesResponse) GetConversations() []*ConvertedConversation { + if x != nil { + return x.Conversations + } + return nil +} + +func (x *ConvertTracesResponse) GetWarnings() []string { + if x != nil { + return x.Warnings + } + return nil +} + +var File_agentevals_api_v1alpha1_conversion_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_conversion_proto_rawDesc = "" + + "\n" + + "(agentevals/api/v1alpha1/conversion.proto\x12\x17agentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a#agentevals/api/v1alpha1/views.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x90\x01\n" + + "\x14ConvertTracesRequest\x12;\n" + + "\x06traces\x18\x01 \x01(\v2#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n" + + "\bgroup_by\x18\x02 \x01(\x0e2 .agentevals.api.v1alpha1.GroupByR\agroupBy\"\xae\x01\n" + + "\x15ConvertedConversation\x12M\n" + + "\fconversation\x18\x01 \x01(\v2).agentevals.api.v1alpha1.ConversationViewR\fconversation\x12F\n" + + "\x13golden_adk_eval_set\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x10goldenAdkEvalSet\"\x89\x01\n" + + "\x15ConvertTracesResponse\x12T\n" + + "\rconversations\x18\x01 \x03(\v2..agentevals.api.v1alpha1.ConvertedConversationR\rconversations\x12\x1a\n" + + "\bwarnings\x18\x02 \x03(\tR\bwarnings2\x83\x01\n" + + "\x11ConversionService\x12n\n" + + "\rConvertTraces\x12-.agentevals.api.v1alpha1.ConvertTracesRequest\x1a..agentevals.api.v1alpha1.ConvertTracesResponseBQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_conversion_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_conversion_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_conversion_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_conversion_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_conversion_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_conversion_proto_rawDesc), len(file_agentevals_api_v1alpha1_conversion_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_conversion_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_conversion_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_agentevals_api_v1alpha1_conversion_proto_goTypes = []any{ + (*ConvertTracesRequest)(nil), // 0: agentevals.api.v1alpha1.ConvertTracesRequest + (*ConvertedConversation)(nil), // 1: agentevals.api.v1alpha1.ConvertedConversation + (*ConvertTracesResponse)(nil), // 2: agentevals.api.v1alpha1.ConvertTracesResponse + (*TraceInput)(nil), // 3: agentevals.api.v1alpha1.TraceInput + (GroupBy)(0), // 4: agentevals.api.v1alpha1.GroupBy + (*ConversationView)(nil), // 5: agentevals.api.v1alpha1.ConversationView + (*structpb.Struct)(nil), // 6: google.protobuf.Struct +} +var file_agentevals_api_v1alpha1_conversion_proto_depIdxs = []int32{ + 3, // 0: agentevals.api.v1alpha1.ConvertTracesRequest.traces:type_name -> agentevals.api.v1alpha1.TraceInput + 4, // 1: agentevals.api.v1alpha1.ConvertTracesRequest.group_by:type_name -> agentevals.api.v1alpha1.GroupBy + 5, // 2: agentevals.api.v1alpha1.ConvertedConversation.conversation:type_name -> agentevals.api.v1alpha1.ConversationView + 6, // 3: agentevals.api.v1alpha1.ConvertedConversation.golden_adk_eval_set:type_name -> google.protobuf.Struct + 1, // 4: agentevals.api.v1alpha1.ConvertTracesResponse.conversations:type_name -> agentevals.api.v1alpha1.ConvertedConversation + 0, // 5: agentevals.api.v1alpha1.ConversionService.ConvertTraces:input_type -> agentevals.api.v1alpha1.ConvertTracesRequest + 2, // 6: agentevals.api.v1alpha1.ConversionService.ConvertTraces:output_type -> agentevals.api.v1alpha1.ConvertTracesResponse + 6, // [6:7] is the sub-list for method output_type + 5, // [5:6] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_conversion_proto_init() } +func file_agentevals_api_v1alpha1_conversion_proto_init() { + if File_agentevals_api_v1alpha1_conversion_proto != nil { + return + } + file_agentevals_api_v1alpha1_common_proto_init() + file_agentevals_api_v1alpha1_views_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_conversion_proto_rawDesc), len(file_agentevals_api_v1alpha1_conversion_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agentevals_api_v1alpha1_conversion_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_conversion_proto_depIdxs, + MessageInfos: file_agentevals_api_v1alpha1_conversion_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_conversion_proto = out.File + file_agentevals_api_v1alpha1_conversion_proto_goTypes = nil + file_agentevals_api_v1alpha1_conversion_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/conversion_grpc.pb.go b/gen/go/agentevals/api/v1alpha1/conversion_grpc.pb.go new file mode 100644 index 00000000..322918b4 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/conversion_grpc.pb.go @@ -0,0 +1,123 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: agentevals/api/v1alpha1/conversion.proto + +package apiv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ConversionService_ConvertTraces_FullMethodName = "/agentevals.api.v1alpha1.ConversionService/ConvertTraces" +) + +// ConversionServiceClient is the client API for ConversionService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ConversionServiceClient interface { + // Reads traces into conversations without evaluating them. + ConvertTraces(ctx context.Context, in *ConvertTracesRequest, opts ...grpc.CallOption) (*ConvertTracesResponse, error) +} + +type conversionServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewConversionServiceClient(cc grpc.ClientConnInterface) ConversionServiceClient { + return &conversionServiceClient{cc} +} + +func (c *conversionServiceClient) ConvertTraces(ctx context.Context, in *ConvertTracesRequest, opts ...grpc.CallOption) (*ConvertTracesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ConvertTracesResponse) + err := c.cc.Invoke(ctx, ConversionService_ConvertTraces_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ConversionServiceServer is the server API for ConversionService service. +// All implementations must embed UnimplementedConversionServiceServer +// for forward compatibility. +type ConversionServiceServer interface { + // Reads traces into conversations without evaluating them. + ConvertTraces(context.Context, *ConvertTracesRequest) (*ConvertTracesResponse, error) + mustEmbedUnimplementedConversionServiceServer() +} + +// UnimplementedConversionServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedConversionServiceServer struct{} + +func (UnimplementedConversionServiceServer) ConvertTraces(context.Context, *ConvertTracesRequest) (*ConvertTracesResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ConvertTraces not implemented") +} +func (UnimplementedConversionServiceServer) mustEmbedUnimplementedConversionServiceServer() {} +func (UnimplementedConversionServiceServer) testEmbeddedByValue() {} + +// UnsafeConversionServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ConversionServiceServer will +// result in compilation errors. +type UnsafeConversionServiceServer interface { + mustEmbedUnimplementedConversionServiceServer() +} + +func RegisterConversionServiceServer(s grpc.ServiceRegistrar, srv ConversionServiceServer) { + // If the following call panics, it indicates UnimplementedConversionServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ConversionService_ServiceDesc, srv) +} + +func _ConversionService_ConvertTraces_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConvertTracesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ConversionServiceServer).ConvertTraces(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ConversionService_ConvertTraces_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ConversionServiceServer).ConvertTraces(ctx, req.(*ConvertTracesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ConversionService_ServiceDesc is the grpc.ServiceDesc for ConversionService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ConversionService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agentevals.api.v1alpha1.ConversionService", + HandlerType: (*ConversionServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ConvertTraces", + Handler: _ConversionService_ConvertTraces_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "agentevals/api/v1alpha1/conversion.proto", +} diff --git a/gen/go/agentevals/api/v1alpha1/evaluation.pb.go b/gen/go/agentevals/api/v1alpha1/evaluation.pb.go new file mode 100644 index 00000000..03757e60 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/evaluation.pb.go @@ -0,0 +1,1777 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/evaluation.proto + +package apiv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TrajectoryMatchType int32 + +const ( + // The metric's default. + TrajectoryMatchType_TRAJECTORY_MATCH_TYPE_UNSPECIFIED TrajectoryMatchType = 0 + TrajectoryMatchType_TRAJECTORY_MATCH_TYPE_EXACT TrajectoryMatchType = 1 + TrajectoryMatchType_TRAJECTORY_MATCH_TYPE_IN_ORDER TrajectoryMatchType = 2 + TrajectoryMatchType_TRAJECTORY_MATCH_TYPE_ANY_ORDER TrajectoryMatchType = 3 +) + +// Enum value maps for TrajectoryMatchType. +var ( + TrajectoryMatchType_name = map[int32]string{ + 0: "TRAJECTORY_MATCH_TYPE_UNSPECIFIED", + 1: "TRAJECTORY_MATCH_TYPE_EXACT", + 2: "TRAJECTORY_MATCH_TYPE_IN_ORDER", + 3: "TRAJECTORY_MATCH_TYPE_ANY_ORDER", + } + TrajectoryMatchType_value = map[string]int32{ + "TRAJECTORY_MATCH_TYPE_UNSPECIFIED": 0, + "TRAJECTORY_MATCH_TYPE_EXACT": 1, + "TRAJECTORY_MATCH_TYPE_IN_ORDER": 2, + "TRAJECTORY_MATCH_TYPE_ANY_ORDER": 3, + } +) + +func (x TrajectoryMatchType) Enum() *TrajectoryMatchType { + p := new(TrajectoryMatchType) + *p = x + return p +} + +func (x TrajectoryMatchType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TrajectoryMatchType) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_evaluation_proto_enumTypes[0].Descriptor() +} + +func (TrajectoryMatchType) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_evaluation_proto_enumTypes[0] +} + +func (x TrajectoryMatchType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TrajectoryMatchType.Descriptor instead. +func (TrajectoryMatchType) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{0} +} + +type EvalStatus int32 + +const ( + EvalStatus_EVAL_STATUS_UNSPECIFIED EvalStatus = 0 + EvalStatus_EVAL_STATUS_PASSED EvalStatus = 1 + EvalStatus_EVAL_STATUS_FAILED EvalStatus = 2 + EvalStatus_EVAL_STATUS_NOT_EVALUATED EvalStatus = 3 +) + +// Enum value maps for EvalStatus. +var ( + EvalStatus_name = map[int32]string{ + 0: "EVAL_STATUS_UNSPECIFIED", + 1: "EVAL_STATUS_PASSED", + 2: "EVAL_STATUS_FAILED", + 3: "EVAL_STATUS_NOT_EVALUATED", + } + EvalStatus_value = map[string]int32{ + "EVAL_STATUS_UNSPECIFIED": 0, + "EVAL_STATUS_PASSED": 1, + "EVAL_STATUS_FAILED": 2, + "EVAL_STATUS_NOT_EVALUATED": 3, + } +) + +func (x EvalStatus) Enum() *EvalStatus { + p := new(EvalStatus) + *p = x + return p +} + +func (x EvalStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EvalStatus) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_evaluation_proto_enumTypes[1].Descriptor() +} + +func (EvalStatus) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_evaluation_proto_enumTypes[1] +} + +func (x EvalStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EvalStatus.Descriptor instead. +func (EvalStatus) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{1} +} + +type ListMetricsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListMetricsRequest) Reset() { + *x = ListMetricsRequest{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListMetricsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListMetricsRequest) ProtoMessage() {} + +func (x *ListMetricsRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListMetricsRequest.ProtoReflect.Descriptor instead. +func (*ListMetricsRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{0} +} + +type Metric struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Category string `protobuf:"bytes,2,opt,name=category,proto3" json:"category,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + // Needs a golden eval set. + RequiresEvalSet bool `protobuf:"varint,4,opt,name=requires_eval_set,json=requiresEvalSet,proto3" json:"requires_eval_set,omitempty"` + // Calls a judge model. + RequiresLlm bool `protobuf:"varint,5,opt,name=requires_llm,json=requiresLlm,proto3" json:"requires_llm,omitempty"` + // Needs Google Cloud credentials. + RequiresGcp bool `protobuf:"varint,6,opt,name=requires_gcp,json=requiresGcp,proto3" json:"requires_gcp,omitempty"` + // Needs rubrics in the eval set. + RequiresRubrics bool `protobuf:"varint,7,opt,name=requires_rubrics,json=requiresRubrics,proto3" json:"requires_rubrics,omitempty"` + // Usable with recorded traces. + Working bool `protobuf:"varint,8,opt,name=working,proto3" json:"working,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Metric) Reset() { + *x = Metric{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Metric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metric) ProtoMessage() {} + +func (x *Metric) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metric.ProtoReflect.Descriptor instead. +func (*Metric) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{1} +} + +func (x *Metric) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Metric) GetCategory() string { + if x != nil { + return x.Category + } + return "" +} + +func (x *Metric) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *Metric) GetRequiresEvalSet() bool { + if x != nil { + return x.RequiresEvalSet + } + return false +} + +func (x *Metric) GetRequiresLlm() bool { + if x != nil { + return x.RequiresLlm + } + return false +} + +func (x *Metric) GetRequiresGcp() bool { + if x != nil { + return x.RequiresGcp + } + return false +} + +func (x *Metric) GetRequiresRubrics() bool { + if x != nil { + return x.RequiresRubrics + } + return false +} + +func (x *Metric) GetWorking() bool { + if x != nil { + return x.Working + } + return false +} + +type ListMetricsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metrics []*Metric `protobuf:"bytes,1,rep,name=metrics,proto3" json:"metrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListMetricsResponse) Reset() { + *x = ListMetricsResponse{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListMetricsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListMetricsResponse) ProtoMessage() {} + +func (x *ListMetricsResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListMetricsResponse.ProtoReflect.Descriptor instead. +func (*ListMetricsResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{2} +} + +func (x *ListMetricsResponse) GetMetrics() []*Metric { + if x != nil { + return x.Metrics + } + return nil +} + +// A built-in metric, by name (see ListMetrics), with optional overrides. +type BuiltinEvaluator struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // 0 to 1. Unset uses the metric's default. + Threshold *float64 `protobuf:"fixed64,2,opt,name=threshold,proto3,oneof" json:"threshold,omitempty"` + JudgeModel string `protobuf:"bytes,3,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` + TrajectoryMatchType TrajectoryMatchType `protobuf:"varint,4,opt,name=trajectory_match_type,json=trajectoryMatchType,proto3,enum=agentevals.api.v1alpha1.TrajectoryMatchType" json:"trajectory_match_type,omitempty"` + // Name of a credential_refs entry whose value is the judge API key. + CredentialRef string `protobuf:"bytes,5,opt,name=credential_ref,json=credentialRef,proto3" json:"credential_ref,omitempty"` + // Base URL of an OpenAI compatible judge endpoint. + JudgeBaseUrl string `protobuf:"bytes,6,opt,name=judge_base_url,json=judgeBaseUrl,proto3" json:"judge_base_url,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BuiltinEvaluator) Reset() { + *x = BuiltinEvaluator{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BuiltinEvaluator) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BuiltinEvaluator) ProtoMessage() {} + +func (x *BuiltinEvaluator) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BuiltinEvaluator.ProtoReflect.Descriptor instead. +func (*BuiltinEvaluator) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{3} +} + +func (x *BuiltinEvaluator) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *BuiltinEvaluator) GetThreshold() float64 { + if x != nil && x.Threshold != nil { + return *x.Threshold + } + return 0 +} + +func (x *BuiltinEvaluator) GetJudgeModel() string { + if x != nil { + return x.JudgeModel + } + return "" +} + +func (x *BuiltinEvaluator) GetTrajectoryMatchType() TrajectoryMatchType { + if x != nil { + return x.TrajectoryMatchType + } + return TrajectoryMatchType_TRAJECTORY_MATCH_TYPE_UNSPECIFIED +} + +func (x *BuiltinEvaluator) GetCredentialRef() string { + if x != nil { + return x.CredentialRef + } + return "" +} + +func (x *BuiltinEvaluator) GetJudgeBaseUrl() string { + if x != nil { + return x.JudgeBaseUrl + } + return "" +} + +// An evaluator file on the server, run as a subprocess. +type CodeEvaluator struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // 0 to 1. Unset means 0.5. + Threshold *float64 `protobuf:"fixed64,2,opt,name=threshold,proto3,oneof" json:"threshold,omitempty"` + // Seconds. Unset means 30. + Timeout *int32 `protobuf:"varint,3,opt,name=timeout,proto3,oneof" json:"timeout,omitempty"` + Config *structpb.Struct `protobuf:"bytes,4,opt,name=config,proto3" json:"config,omitempty"` + // Unset means local. + Executor string `protobuf:"bytes,5,opt,name=executor,proto3" json:"executor,omitempty"` + // Path of the evaluator file (.py, .js, .ts and so on) on the server. + Path string `protobuf:"bytes,6,opt,name=path,proto3" json:"path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CodeEvaluator) Reset() { + *x = CodeEvaluator{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CodeEvaluator) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CodeEvaluator) ProtoMessage() {} + +func (x *CodeEvaluator) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CodeEvaluator.ProtoReflect.Descriptor instead. +func (*CodeEvaluator) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{4} +} + +func (x *CodeEvaluator) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CodeEvaluator) GetThreshold() float64 { + if x != nil && x.Threshold != nil { + return *x.Threshold + } + return 0 +} + +func (x *CodeEvaluator) GetTimeout() int32 { + if x != nil && x.Timeout != nil { + return *x.Timeout + } + return 0 +} + +func (x *CodeEvaluator) GetConfig() *structpb.Struct { + if x != nil { + return x.Config + } + return nil +} + +func (x *CodeEvaluator) GetExecutor() string { + if x != nil { + return x.Executor + } + return "" +} + +func (x *CodeEvaluator) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +// An evaluator fetched from a remote source. +type RemoteEvaluator struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // 0 to 1. Unset means 0.5. + Threshold *float64 `protobuf:"fixed64,2,opt,name=threshold,proto3,oneof" json:"threshold,omitempty"` + // Seconds. Unset means 30. + Timeout *int32 `protobuf:"varint,3,opt,name=timeout,proto3,oneof" json:"timeout,omitempty"` + Config *structpb.Struct `protobuf:"bytes,4,opt,name=config,proto3" json:"config,omitempty"` + // Unset means local. + Executor string `protobuf:"bytes,5,opt,name=executor,proto3" json:"executor,omitempty"` + // Unset means github. + Source string `protobuf:"bytes,6,opt,name=source,proto3" json:"source,omitempty"` + // A relative path within the source. + Ref string `protobuf:"bytes,7,opt,name=ref,proto3" json:"ref,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RemoteEvaluator) Reset() { + *x = RemoteEvaluator{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RemoteEvaluator) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteEvaluator) ProtoMessage() {} + +func (x *RemoteEvaluator) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteEvaluator.ProtoReflect.Descriptor instead. +func (*RemoteEvaluator) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{5} +} + +func (x *RemoteEvaluator) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RemoteEvaluator) GetThreshold() float64 { + if x != nil && x.Threshold != nil { + return *x.Threshold + } + return 0 +} + +func (x *RemoteEvaluator) GetTimeout() int32 { + if x != nil && x.Timeout != nil { + return *x.Timeout + } + return 0 +} + +func (x *RemoteEvaluator) GetConfig() *structpb.Struct { + if x != nil { + return x.Config + } + return nil +} + +func (x *RemoteEvaluator) GetExecutor() string { + if x != nil { + return x.Executor + } + return "" +} + +func (x *RemoteEvaluator) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *RemoteEvaluator) GetRef() string { + if x != nil { + return x.Ref + } + return "" +} + +type Evaluator struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: + // + // *Evaluator_Builtin + // *Evaluator_Code + // *Evaluator_Remote + Kind isEvaluator_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Evaluator) Reset() { + *x = Evaluator{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Evaluator) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Evaluator) ProtoMessage() {} + +func (x *Evaluator) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Evaluator.ProtoReflect.Descriptor instead. +func (*Evaluator) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{6} +} + +func (x *Evaluator) GetKind() isEvaluator_Kind { + if x != nil { + return x.Kind + } + return nil +} + +func (x *Evaluator) GetBuiltin() *BuiltinEvaluator { + if x != nil { + if x, ok := x.Kind.(*Evaluator_Builtin); ok { + return x.Builtin + } + } + return nil +} + +func (x *Evaluator) GetCode() *CodeEvaluator { + if x != nil { + if x, ok := x.Kind.(*Evaluator_Code); ok { + return x.Code + } + } + return nil +} + +func (x *Evaluator) GetRemote() *RemoteEvaluator { + if x != nil { + if x, ok := x.Kind.(*Evaluator_Remote); ok { + return x.Remote + } + } + return nil +} + +type isEvaluator_Kind interface { + isEvaluator_Kind() +} + +type Evaluator_Builtin struct { + Builtin *BuiltinEvaluator `protobuf:"bytes,1,opt,name=builtin,proto3,oneof"` +} + +type Evaluator_Code struct { + Code *CodeEvaluator `protobuf:"bytes,2,opt,name=code,proto3,oneof"` +} + +type Evaluator_Remote struct { + Remote *RemoteEvaluator `protobuf:"bytes,3,opt,name=remote,proto3,oneof"` +} + +func (*Evaluator_Builtin) isEvaluator_Kind() {} + +func (*Evaluator_Code) isEvaluator_Kind() {} + +func (*Evaluator_Remote) isEvaluator_Kind() {} + +type EvalParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Names must be unique. Empty means tool_trajectory_avg_score. + Evaluators []*Evaluator `protobuf:"bytes,1,rep,name=evaluators,proto3" json:"evaluators,omitempty"` + // At least 1. Unset means 10. + MaxConcurrentTraces *int32 `protobuf:"varint,2,opt,name=max_concurrent_traces,json=maxConcurrentTraces,proto3,oneof" json:"max_concurrent_traces,omitempty"` + // At least 1. Unset means 5. + MaxConcurrentEvals *int32 `protobuf:"varint,3,opt,name=max_concurrent_evals,json=maxConcurrentEvals,proto3,oneof" json:"max_concurrent_evals,omitempty"` + // Unset means GROUP_BY_AUTO. + GroupBy GroupBy `protobuf:"varint,4,opt,name=group_by,json=groupBy,proto3,enum=agentevals.api.v1alpha1.GroupBy" json:"group_by,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvalParams) Reset() { + *x = EvalParams{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvalParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvalParams) ProtoMessage() {} + +func (x *EvalParams) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvalParams.ProtoReflect.Descriptor instead. +func (*EvalParams) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{7} +} + +func (x *EvalParams) GetEvaluators() []*Evaluator { + if x != nil { + return x.Evaluators + } + return nil +} + +func (x *EvalParams) GetMaxConcurrentTraces() int32 { + if x != nil && x.MaxConcurrentTraces != nil { + return *x.MaxConcurrentTraces + } + return 0 +} + +func (x *EvalParams) GetMaxConcurrentEvals() int32 { + if x != nil && x.MaxConcurrentEvals != nil { + return *x.MaxConcurrentEvals + } + return 0 +} + +func (x *EvalParams) GetGroupBy() GroupBy { + if x != nil { + return x.GroupBy + } + return GroupBy_GROUP_BY_UNSPECIFIED +} + +// A metric's result for one turn. +type InvocationResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Unset when the turn has no score. + Score *float64 `protobuf:"fixed64,1,opt,name=score,proto3,oneof" json:"score,omitempty"` + Status EvalStatus `protobuf:"varint,2,opt,name=status,proto3,enum=agentevals.api.v1alpha1.EvalStatus" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InvocationResult) Reset() { + *x = InvocationResult{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InvocationResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InvocationResult) ProtoMessage() {} + +func (x *InvocationResult) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InvocationResult.ProtoReflect.Descriptor instead. +func (*InvocationResult) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{8} +} + +func (x *InvocationResult) GetScore() float64 { + if x != nil && x.Score != nil { + return *x.Score + } + return 0 +} + +func (x *InvocationResult) GetStatus() EvalStatus { + if x != nil { + return x.Status + } + return EvalStatus_EVAL_STATUS_UNSPECIFIED +} + +type MetricResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + MetricName string `protobuf:"bytes,1,opt,name=metric_name,json=metricName,proto3" json:"metric_name,omitempty"` + Score *float64 `protobuf:"fixed64,2,opt,name=score,proto3,oneof" json:"score,omitempty"` + EvalStatus EvalStatus `protobuf:"varint,3,opt,name=eval_status,json=evalStatus,proto3,enum=agentevals.api.v1alpha1.EvalStatus" json:"eval_status,omitempty"` + // One entry per turn, in turn order. + PerInvocation []*InvocationResult `protobuf:"bytes,4,rep,name=per_invocation,json=perInvocation,proto3" json:"per_invocation,omitempty"` + Error *string `protobuf:"bytes,5,opt,name=error,proto3,oneof" json:"error,omitempty"` + // error.type. + ErrorType *string `protobuf:"bytes,6,opt,name=error_type,json=errorType,proto3,oneof" json:"error_type,omitempty"` + // Metric specific details. + Details *structpb.Struct `protobuf:"bytes,7,opt,name=details,proto3" json:"details,omitempty"` + DurationMs *float64 `protobuf:"fixed64,8,opt,name=duration_ms,json=durationMs,proto3,oneof" json:"duration_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricResult) Reset() { + *x = MetricResult{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricResult) ProtoMessage() {} + +func (x *MetricResult) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricResult.ProtoReflect.Descriptor instead. +func (*MetricResult) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{9} +} + +func (x *MetricResult) GetMetricName() string { + if x != nil { + return x.MetricName + } + return "" +} + +func (x *MetricResult) GetScore() float64 { + if x != nil && x.Score != nil { + return *x.Score + } + return 0 +} + +func (x *MetricResult) GetEvalStatus() EvalStatus { + if x != nil { + return x.EvalStatus + } + return EvalStatus_EVAL_STATUS_UNSPECIFIED +} + +func (x *MetricResult) GetPerInvocation() []*InvocationResult { + if x != nil { + return x.PerInvocation + } + return nil +} + +func (x *MetricResult) GetError() string { + if x != nil && x.Error != nil { + return *x.Error + } + return "" +} + +func (x *MetricResult) GetErrorType() string { + if x != nil && x.ErrorType != nil { + return *x.ErrorType + } + return "" +} + +func (x *MetricResult) GetDetails() *structpb.Struct { + if x != nil { + return x.Details + } + return nil +} + +func (x *MetricResult) GetDurationMs() float64 { + if x != nil && x.DurationMs != nil { + return *x.DurationMs + } + return 0 +} + +type TurnRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + TraceId string `protobuf:"bytes,1,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanId string `protobuf:"bytes,2,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TurnRef) Reset() { + *x = TurnRef{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TurnRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TurnRef) ProtoMessage() {} + +func (x *TurnRef) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TurnRef.ProtoReflect.Descriptor instead. +func (*TurnRef) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{10} +} + +func (x *TurnRef) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *TurnRef) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +// The results of one evaluation group: a trace, or a conversation of several traces. +type TraceResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The group's earliest trace. + TraceId string `protobuf:"bytes,1,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + NumInvocations int32 `protobuf:"varint,2,opt,name=num_invocations,json=numInvocations,proto3" json:"num_invocations,omitempty"` + MetricResults []*MetricResult `protobuf:"bytes,3,rep,name=metric_results,json=metricResults,proto3" json:"metric_results,omitempty"` + ConversionWarnings []string `protobuf:"bytes,4,rep,name=conversion_warnings,json=conversionWarnings,proto3" json:"conversion_warnings,omitempty"` + PerformanceMetrics *structpb.Struct `protobuf:"bytes,5,opt,name=performance_metrics,json=performanceMetrics,proto3" json:"performance_metrics,omitempty"` + // service.name. + ServiceName string `protobuf:"bytes,6,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` + // gen_ai.agent.name. + AgentName string `protobuf:"bytes,7,opt,name=agent_name,json=agentName,proto3" json:"agent_name,omitempty"` + TraceIds []string `protobuf:"bytes,8,rep,name=trace_ids,json=traceIds,proto3" json:"trace_ids,omitempty"` + // gen_ai.conversation.id. + ConversationId string `protobuf:"bytes,9,opt,name=conversation_id,json=conversationId,proto3" json:"conversation_id,omitempty"` + GroupKey string `protobuf:"bytes,10,opt,name=group_key,json=groupKey,proto3" json:"group_key,omitempty"` + // The eval set case the group was compared with. + EvalCaseId string `protobuf:"bytes,11,opt,name=eval_case_id,json=evalCaseId,proto3" json:"eval_case_id,omitempty"` + // How the case was chosen. + EvalCaseMatch string `protobuf:"bytes,12,opt,name=eval_case_match,json=evalCaseMatch,proto3" json:"eval_case_match,omitempty"` + // The judged turns, in order. + TurnRefs []*TurnRef `protobuf:"bytes,13,rep,name=turn_refs,json=turnRefs,proto3" json:"turn_refs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TraceResult) Reset() { + *x = TraceResult{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceResult) ProtoMessage() {} + +func (x *TraceResult) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceResult.ProtoReflect.Descriptor instead. +func (*TraceResult) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{11} +} + +func (x *TraceResult) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *TraceResult) GetNumInvocations() int32 { + if x != nil { + return x.NumInvocations + } + return 0 +} + +func (x *TraceResult) GetMetricResults() []*MetricResult { + if x != nil { + return x.MetricResults + } + return nil +} + +func (x *TraceResult) GetConversionWarnings() []string { + if x != nil { + return x.ConversionWarnings + } + return nil +} + +func (x *TraceResult) GetPerformanceMetrics() *structpb.Struct { + if x != nil { + return x.PerformanceMetrics + } + return nil +} + +func (x *TraceResult) GetServiceName() string { + if x != nil { + return x.ServiceName + } + return "" +} + +func (x *TraceResult) GetAgentName() string { + if x != nil { + return x.AgentName + } + return "" +} + +func (x *TraceResult) GetTraceIds() []string { + if x != nil { + return x.TraceIds + } + return nil +} + +func (x *TraceResult) GetConversationId() string { + if x != nil { + return x.ConversationId + } + return "" +} + +func (x *TraceResult) GetGroupKey() string { + if x != nil { + return x.GroupKey + } + return "" +} + +func (x *TraceResult) GetEvalCaseId() string { + if x != nil { + return x.EvalCaseId + } + return "" +} + +func (x *TraceResult) GetEvalCaseMatch() string { + if x != nil { + return x.EvalCaseMatch + } + return "" +} + +func (x *TraceResult) GetTurnRefs() []*TurnRef { + if x != nil { + return x.TurnRefs + } + return nil +} + +type RunResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + TraceResults []*TraceResult `protobuf:"bytes,1,rep,name=trace_results,json=traceResults,proto3" json:"trace_results,omitempty"` + Errors []string `protobuf:"bytes,2,rep,name=errors,proto3" json:"errors,omitempty"` + PerformanceMetrics *structpb.Struct `protobuf:"bytes,3,opt,name=performance_metrics,json=performanceMetrics,proto3" json:"performance_metrics,omitempty"` + // Set when the evaluation was recorded as a run. + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunResult) Reset() { + *x = RunResult{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunResult) ProtoMessage() {} + +func (x *RunResult) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunResult.ProtoReflect.Descriptor instead. +func (*RunResult) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{12} +} + +func (x *RunResult) GetTraceResults() []*TraceResult { + if x != nil { + return x.TraceResults + } + return nil +} + +func (x *RunResult) GetErrors() []string { + if x != nil { + return x.Errors + } + return nil +} + +func (x *RunResult) GetPerformanceMetrics() *structpb.Struct { + if x != nil { + return x.PerformanceMetrics + } + return nil +} + +func (x *RunResult) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type EvaluateTracesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Traces *TraceInput `protobuf:"bytes,1,opt,name=traces,proto3" json:"traces,omitempty"` + // The golden. Required by metrics with requires_eval_set. + // + // Types that are valid to be assigned to EvalSet: + // + // *EvaluateTracesRequest_AdkEvalSet + EvalSet isEvaluateTracesRequest_EvalSet `protobuf_oneof:"eval_set"` + Params *EvalParams `protobuf:"bytes,3,opt,name=params,proto3" json:"params,omitempty"` + // Logical credential name to a secret reference, for example + // {"kind": "kubernetes", "namespace": "...", "name": "...", "key": "..."} or + // {"kind": "env", "name": "..."}. Values are resolved per request and never returned. + CredentialRefs map[string]*structpb.Struct `protobuf:"bytes,4,rep,name=credential_refs,json=credentialRefs,proto3" json:"credential_refs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluateTracesRequest) Reset() { + *x = EvaluateTracesRequest{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluateTracesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluateTracesRequest) ProtoMessage() {} + +func (x *EvaluateTracesRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluateTracesRequest.ProtoReflect.Descriptor instead. +func (*EvaluateTracesRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{13} +} + +func (x *EvaluateTracesRequest) GetTraces() *TraceInput { + if x != nil { + return x.Traces + } + return nil +} + +func (x *EvaluateTracesRequest) GetEvalSet() isEvaluateTracesRequest_EvalSet { + if x != nil { + return x.EvalSet + } + return nil +} + +func (x *EvaluateTracesRequest) GetAdkEvalSet() *structpb.Struct { + if x != nil { + if x, ok := x.EvalSet.(*EvaluateTracesRequest_AdkEvalSet); ok { + return x.AdkEvalSet + } + } + return nil +} + +func (x *EvaluateTracesRequest) GetParams() *EvalParams { + if x != nil { + return x.Params + } + return nil +} + +func (x *EvaluateTracesRequest) GetCredentialRefs() map[string]*structpb.Struct { + if x != nil { + return x.CredentialRefs + } + return nil +} + +type isEvaluateTracesRequest_EvalSet interface { + isEvaluateTracesRequest_EvalSet() +} + +type EvaluateTracesRequest_AdkEvalSet struct { + // An ADK EvalSet as JSON. + AdkEvalSet *structpb.Struct `protobuf:"bytes,2,opt,name=adk_eval_set,json=adkEvalSet,proto3,oneof"` +} + +func (*EvaluateTracesRequest_AdkEvalSet) isEvaluateTracesRequest_EvalSet() {} + +type EvaluationProgress struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluationProgress) Reset() { + *x = EvaluationProgress{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluationProgress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluationProgress) ProtoMessage() {} + +func (x *EvaluationProgress) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluationProgress.ProtoReflect.Descriptor instead. +func (*EvaluationProgress) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{14} +} + +func (x *EvaluationProgress) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// A group's performance metrics, sent before it is evaluated. +type GroupPerformance struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The group's earliest trace. + TraceId string `protobuf:"bytes,1,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + PerformanceMetrics *structpb.Struct `protobuf:"bytes,2,opt,name=performance_metrics,json=performanceMetrics,proto3" json:"performance_metrics,omitempty"` + TraceMetadata *structpb.Struct `protobuf:"bytes,3,opt,name=trace_metadata,json=traceMetadata,proto3" json:"trace_metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GroupPerformance) Reset() { + *x = GroupPerformance{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GroupPerformance) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GroupPerformance) ProtoMessage() {} + +func (x *GroupPerformance) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GroupPerformance.ProtoReflect.Descriptor instead. +func (*GroupPerformance) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{15} +} + +func (x *GroupPerformance) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *GroupPerformance) GetPerformanceMetrics() *structpb.Struct { + if x != nil { + return x.PerformanceMetrics + } + return nil +} + +func (x *GroupPerformance) GetTraceMetadata() *structpb.Struct { + if x != nil { + return x.TraceMetadata + } + return nil +} + +type EvaluateTracesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Event: + // + // *EvaluateTracesResponse_Progress + // *EvaluateTracesResponse_Performance + // *EvaluateTracesResponse_TraceResult + // *EvaluateTracesResponse_Done + Event isEvaluateTracesResponse_Event `protobuf_oneof:"event"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluateTracesResponse) Reset() { + *x = EvaluateTracesResponse{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluateTracesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluateTracesResponse) ProtoMessage() {} + +func (x *EvaluateTracesResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluateTracesResponse.ProtoReflect.Descriptor instead. +func (*EvaluateTracesResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{16} +} + +func (x *EvaluateTracesResponse) GetEvent() isEvaluateTracesResponse_Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *EvaluateTracesResponse) GetProgress() *EvaluationProgress { + if x != nil { + if x, ok := x.Event.(*EvaluateTracesResponse_Progress); ok { + return x.Progress + } + } + return nil +} + +func (x *EvaluateTracesResponse) GetPerformance() *GroupPerformance { + if x != nil { + if x, ok := x.Event.(*EvaluateTracesResponse_Performance); ok { + return x.Performance + } + } + return nil +} + +func (x *EvaluateTracesResponse) GetTraceResult() *TraceResult { + if x != nil { + if x, ok := x.Event.(*EvaluateTracesResponse_TraceResult); ok { + return x.TraceResult + } + } + return nil +} + +func (x *EvaluateTracesResponse) GetDone() *RunResult { + if x != nil { + if x, ok := x.Event.(*EvaluateTracesResponse_Done); ok { + return x.Done + } + } + return nil +} + +type isEvaluateTracesResponse_Event interface { + isEvaluateTracesResponse_Event() +} + +type EvaluateTracesResponse_Progress struct { + Progress *EvaluationProgress `protobuf:"bytes,1,opt,name=progress,proto3,oneof"` +} + +type EvaluateTracesResponse_Performance struct { + Performance *GroupPerformance `protobuf:"bytes,2,opt,name=performance,proto3,oneof"` +} + +type EvaluateTracesResponse_TraceResult struct { + // A group's results so far. + TraceResult *TraceResult `protobuf:"bytes,3,opt,name=trace_result,json=traceResult,proto3,oneof"` +} + +type EvaluateTracesResponse_Done struct { + // The final result. + Done *RunResult `protobuf:"bytes,4,opt,name=done,proto3,oneof"` +} + +func (*EvaluateTracesResponse_Progress) isEvaluateTracesResponse_Event() {} + +func (*EvaluateTracesResponse_Performance) isEvaluateTracesResponse_Event() {} + +func (*EvaluateTracesResponse_TraceResult) isEvaluateTracesResponse_Event() {} + +func (*EvaluateTracesResponse_Done) isEvaluateTracesResponse_Event() {} + +var File_agentevals_api_v1alpha1_evaluation_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_evaluation_proto_rawDesc = "" + + "\n" + + "(agentevals/api/v1alpha1/evaluation.proto\x12\x17agentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x14\n" + + "\x12ListMetricsRequest\"\x91\x02\n" + + "\x06Metric\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + + "\bcategory\x18\x02 \x01(\tR\bcategory\x12 \n" + + "\vdescription\x18\x03 \x01(\tR\vdescription\x12*\n" + + "\x11requires_eval_set\x18\x04 \x01(\bR\x0frequiresEvalSet\x12!\n" + + "\frequires_llm\x18\x05 \x01(\bR\vrequiresLlm\x12!\n" + + "\frequires_gcp\x18\x06 \x01(\bR\vrequiresGcp\x12)\n" + + "\x10requires_rubrics\x18\a \x01(\bR\x0frequiresRubrics\x12\x18\n" + + "\aworking\x18\b \x01(\bR\aworking\"P\n" + + "\x13ListMetricsResponse\x129\n" + + "\ametrics\x18\x01 \x03(\v2\x1f.agentevals.api.v1alpha1.MetricR\ametrics\"\xa7\x02\n" + + "\x10BuiltinEvaluator\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + + "\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1f\n" + + "\vjudge_model\x18\x03 \x01(\tR\n" + + "judgeModel\x12`\n" + + "\x15trajectory_match_type\x18\x04 \x01(\x0e2,.agentevals.api.v1alpha1.TrajectoryMatchTypeR\x13trajectoryMatchType\x12%\n" + + "\x0ecredential_ref\x18\x05 \x01(\tR\rcredentialRef\x12$\n" + + "\x0ejudge_base_url\x18\x06 \x01(\tR\fjudgeBaseUrlB\f\n" + + "\n" + + "_threshold\"\xe0\x01\n" + + "\rCodeEvaluator\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + + "\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n" + + "\atimeout\x18\x03 \x01(\x05H\x01R\atimeout\x88\x01\x01\x12/\n" + + "\x06config\x18\x04 \x01(\v2\x17.google.protobuf.StructR\x06config\x12\x1a\n" + + "\bexecutor\x18\x05 \x01(\tR\bexecutor\x12\x12\n" + + "\x04path\x18\x06 \x01(\tR\x04pathB\f\n" + + "\n" + + "_thresholdB\n" + + "\n" + + "\b_timeout\"\xf8\x01\n" + + "\x0fRemoteEvaluator\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + + "\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n" + + "\atimeout\x18\x03 \x01(\x05H\x01R\atimeout\x88\x01\x01\x12/\n" + + "\x06config\x18\x04 \x01(\v2\x17.google.protobuf.StructR\x06config\x12\x1a\n" + + "\bexecutor\x18\x05 \x01(\tR\bexecutor\x12\x16\n" + + "\x06source\x18\x06 \x01(\tR\x06source\x12\x10\n" + + "\x03ref\x18\a \x01(\tR\x03refB\f\n" + + "\n" + + "_thresholdB\n" + + "\n" + + "\b_timeout\"\xdc\x01\n" + + "\tEvaluator\x12E\n" + + "\abuiltin\x18\x01 \x01(\v2).agentevals.api.v1alpha1.BuiltinEvaluatorH\x00R\abuiltin\x12<\n" + + "\x04code\x18\x02 \x01(\v2&.agentevals.api.v1alpha1.CodeEvaluatorH\x00R\x04code\x12B\n" + + "\x06remote\x18\x03 \x01(\v2(.agentevals.api.v1alpha1.RemoteEvaluatorH\x00R\x06remoteB\x06\n" + + "\x04kind\"\xb0\x02\n" + + "\n" + + "EvalParams\x12B\n" + + "\n" + + "evaluators\x18\x01 \x03(\v2\".agentevals.api.v1alpha1.EvaluatorR\n" + + "evaluators\x127\n" + + "\x15max_concurrent_traces\x18\x02 \x01(\x05H\x00R\x13maxConcurrentTraces\x88\x01\x01\x125\n" + + "\x14max_concurrent_evals\x18\x03 \x01(\x05H\x01R\x12maxConcurrentEvals\x88\x01\x01\x12;\n" + + "\bgroup_by\x18\x04 \x01(\x0e2 .agentevals.api.v1alpha1.GroupByR\agroupByB\x18\n" + + "\x16_max_concurrent_tracesB\x17\n" + + "\x15_max_concurrent_evals\"t\n" + + "\x10InvocationResult\x12\x19\n" + + "\x05score\x18\x01 \x01(\x01H\x00R\x05score\x88\x01\x01\x12;\n" + + "\x06status\x18\x02 \x01(\x0e2#.agentevals.api.v1alpha1.EvalStatusR\x06statusB\b\n" + + "\x06_score\"\xad\x03\n" + + "\fMetricResult\x12\x1f\n" + + "\vmetric_name\x18\x01 \x01(\tR\n" + + "metricName\x12\x19\n" + + "\x05score\x18\x02 \x01(\x01H\x00R\x05score\x88\x01\x01\x12D\n" + + "\veval_status\x18\x03 \x01(\x0e2#.agentevals.api.v1alpha1.EvalStatusR\n" + + "evalStatus\x12P\n" + + "\x0eper_invocation\x18\x04 \x03(\v2).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n" + + "\x05error\x18\x05 \x01(\tH\x01R\x05error\x88\x01\x01\x12\"\n" + + "\n" + + "error_type\x18\x06 \x01(\tH\x02R\terrorType\x88\x01\x01\x121\n" + + "\adetails\x18\a \x01(\v2\x17.google.protobuf.StructR\adetails\x12$\n" + + "\vduration_ms\x18\b \x01(\x01H\x03R\n" + + "durationMs\x88\x01\x01B\b\n" + + "\x06_scoreB\b\n" + + "\x06_errorB\r\n" + + "\v_error_typeB\x0e\n" + + "\f_duration_ms\"=\n" + + "\aTurnRef\x12\x19\n" + + "\btrace_id\x18\x01 \x01(\tR\atraceId\x12\x17\n" + + "\aspan_id\x18\x02 \x01(\tR\x06spanId\"\xc8\x04\n" + + "\vTraceResult\x12\x19\n" + + "\btrace_id\x18\x01 \x01(\tR\atraceId\x12'\n" + + "\x0fnum_invocations\x18\x02 \x01(\x05R\x0enumInvocations\x12L\n" + + "\x0emetric_results\x18\x03 \x03(\v2%.agentevals.api.v1alpha1.MetricResultR\rmetricResults\x12/\n" + + "\x13conversion_warnings\x18\x04 \x03(\tR\x12conversionWarnings\x12H\n" + + "\x13performance_metrics\x18\x05 \x01(\v2\x17.google.protobuf.StructR\x12performanceMetrics\x12!\n" + + "\fservice_name\x18\x06 \x01(\tR\vserviceName\x12\x1d\n" + + "\n" + + "agent_name\x18\a \x01(\tR\tagentName\x12\x1b\n" + + "\ttrace_ids\x18\b \x03(\tR\btraceIds\x12'\n" + + "\x0fconversation_id\x18\t \x01(\tR\x0econversationId\x12\x1b\n" + + "\tgroup_key\x18\n" + + " \x01(\tR\bgroupKey\x12 \n" + + "\feval_case_id\x18\v \x01(\tR\n" + + "evalCaseId\x12&\n" + + "\x0feval_case_match\x18\f \x01(\tR\revalCaseMatch\x12=\n" + + "\tturn_refs\x18\r \x03(\v2 .agentevals.api.v1alpha1.TurnRefR\bturnRefs\"\xcf\x01\n" + + "\tRunResult\x12I\n" + + "\rtrace_results\x18\x01 \x03(\v2$.agentevals.api.v1alpha1.TraceResultR\ftraceResults\x12\x16\n" + + "\x06errors\x18\x02 \x03(\tR\x06errors\x12H\n" + + "\x13performance_metrics\x18\x03 \x01(\v2\x17.google.protobuf.StructR\x12performanceMetrics\x12\x15\n" + + "\x06run_id\x18\x04 \x01(\tR\x05runId\"\xa3\x03\n" + + "\x15EvaluateTracesRequest\x12;\n" + + "\x06traces\x18\x01 \x01(\v2#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n" + + "\fadk_eval_set\x18\x02 \x01(\v2\x17.google.protobuf.StructH\x00R\n" + + "adkEvalSet\x12;\n" + + "\x06params\x18\x03 \x01(\v2#.agentevals.api.v1alpha1.EvalParamsR\x06params\x12k\n" + + "\x0fcredential_refs\x18\x04 \x03(\v2B.agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntryR\x0ecredentialRefs\x1aZ\n" + + "\x13CredentialRefsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12-\n" + + "\x05value\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x05value:\x028\x01B\n" + + "\n" + + "\beval_set\".\n" + + "\x12EvaluationProgress\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\"\xb7\x01\n" + + "\x10GroupPerformance\x12\x19\n" + + "\btrace_id\x18\x01 \x01(\tR\atraceId\x12H\n" + + "\x13performance_metrics\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x12performanceMetrics\x12>\n" + + "\x0etrace_metadata\x18\x03 \x01(\v2\x17.google.protobuf.StructR\rtraceMetadata\"\xc0\x02\n" + + "\x16EvaluateTracesResponse\x12I\n" + + "\bprogress\x18\x01 \x01(\v2+.agentevals.api.v1alpha1.EvaluationProgressH\x00R\bprogress\x12M\n" + + "\vperformance\x18\x02 \x01(\v2).agentevals.api.v1alpha1.GroupPerformanceH\x00R\vperformance\x12I\n" + + "\ftrace_result\x18\x03 \x01(\v2$.agentevals.api.v1alpha1.TraceResultH\x00R\vtraceResult\x128\n" + + "\x04done\x18\x04 \x01(\v2\".agentevals.api.v1alpha1.RunResultH\x00R\x04doneB\a\n" + + "\x05event*\xa6\x01\n" + + "\x13TrajectoryMatchType\x12%\n" + + "!TRAJECTORY_MATCH_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bTRAJECTORY_MATCH_TYPE_EXACT\x10\x01\x12\"\n" + + "\x1eTRAJECTORY_MATCH_TYPE_IN_ORDER\x10\x02\x12#\n" + + "\x1fTRAJECTORY_MATCH_TYPE_ANY_ORDER\x10\x03*x\n" + + "\n" + + "EvalStatus\x12\x1b\n" + + "\x17EVAL_STATUS_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12EVAL_STATUS_PASSED\x10\x01\x12\x16\n" + + "\x12EVAL_STATUS_FAILED\x10\x02\x12\x1d\n" + + "\x19EVAL_STATUS_NOT_EVALUATED\x10\x032\xf2\x01\n" + + "\x11EvaluationService\x12h\n" + + "\vListMetrics\x12+.agentevals.api.v1alpha1.ListMetricsRequest\x1a,.agentevals.api.v1alpha1.ListMetricsResponse\x12s\n" + + "\x0eEvaluateTraces\x12..agentevals.api.v1alpha1.EvaluateTracesRequest\x1a/.agentevals.api.v1alpha1.EvaluateTracesResponse0\x01BQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_evaluation_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_evaluation_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_evaluation_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_evaluation_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc), len(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_evaluation_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_agentevals_api_v1alpha1_evaluation_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_agentevals_api_v1alpha1_evaluation_proto_goTypes = []any{ + (TrajectoryMatchType)(0), // 0: agentevals.api.v1alpha1.TrajectoryMatchType + (EvalStatus)(0), // 1: agentevals.api.v1alpha1.EvalStatus + (*ListMetricsRequest)(nil), // 2: agentevals.api.v1alpha1.ListMetricsRequest + (*Metric)(nil), // 3: agentevals.api.v1alpha1.Metric + (*ListMetricsResponse)(nil), // 4: agentevals.api.v1alpha1.ListMetricsResponse + (*BuiltinEvaluator)(nil), // 5: agentevals.api.v1alpha1.BuiltinEvaluator + (*CodeEvaluator)(nil), // 6: agentevals.api.v1alpha1.CodeEvaluator + (*RemoteEvaluator)(nil), // 7: agentevals.api.v1alpha1.RemoteEvaluator + (*Evaluator)(nil), // 8: agentevals.api.v1alpha1.Evaluator + (*EvalParams)(nil), // 9: agentevals.api.v1alpha1.EvalParams + (*InvocationResult)(nil), // 10: agentevals.api.v1alpha1.InvocationResult + (*MetricResult)(nil), // 11: agentevals.api.v1alpha1.MetricResult + (*TurnRef)(nil), // 12: agentevals.api.v1alpha1.TurnRef + (*TraceResult)(nil), // 13: agentevals.api.v1alpha1.TraceResult + (*RunResult)(nil), // 14: agentevals.api.v1alpha1.RunResult + (*EvaluateTracesRequest)(nil), // 15: agentevals.api.v1alpha1.EvaluateTracesRequest + (*EvaluationProgress)(nil), // 16: agentevals.api.v1alpha1.EvaluationProgress + (*GroupPerformance)(nil), // 17: agentevals.api.v1alpha1.GroupPerformance + (*EvaluateTracesResponse)(nil), // 18: agentevals.api.v1alpha1.EvaluateTracesResponse + nil, // 19: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry + (*structpb.Struct)(nil), // 20: google.protobuf.Struct + (GroupBy)(0), // 21: agentevals.api.v1alpha1.GroupBy + (*TraceInput)(nil), // 22: agentevals.api.v1alpha1.TraceInput +} +var file_agentevals_api_v1alpha1_evaluation_proto_depIdxs = []int32{ + 3, // 0: agentevals.api.v1alpha1.ListMetricsResponse.metrics:type_name -> agentevals.api.v1alpha1.Metric + 0, // 1: agentevals.api.v1alpha1.BuiltinEvaluator.trajectory_match_type:type_name -> agentevals.api.v1alpha1.TrajectoryMatchType + 20, // 2: agentevals.api.v1alpha1.CodeEvaluator.config:type_name -> google.protobuf.Struct + 20, // 3: agentevals.api.v1alpha1.RemoteEvaluator.config:type_name -> google.protobuf.Struct + 5, // 4: agentevals.api.v1alpha1.Evaluator.builtin:type_name -> agentevals.api.v1alpha1.BuiltinEvaluator + 6, // 5: agentevals.api.v1alpha1.Evaluator.code:type_name -> agentevals.api.v1alpha1.CodeEvaluator + 7, // 6: agentevals.api.v1alpha1.Evaluator.remote:type_name -> agentevals.api.v1alpha1.RemoteEvaluator + 8, // 7: agentevals.api.v1alpha1.EvalParams.evaluators:type_name -> agentevals.api.v1alpha1.Evaluator + 21, // 8: agentevals.api.v1alpha1.EvalParams.group_by:type_name -> agentevals.api.v1alpha1.GroupBy + 1, // 9: agentevals.api.v1alpha1.InvocationResult.status:type_name -> agentevals.api.v1alpha1.EvalStatus + 1, // 10: agentevals.api.v1alpha1.MetricResult.eval_status:type_name -> agentevals.api.v1alpha1.EvalStatus + 10, // 11: agentevals.api.v1alpha1.MetricResult.per_invocation:type_name -> agentevals.api.v1alpha1.InvocationResult + 20, // 12: agentevals.api.v1alpha1.MetricResult.details:type_name -> google.protobuf.Struct + 11, // 13: agentevals.api.v1alpha1.TraceResult.metric_results:type_name -> agentevals.api.v1alpha1.MetricResult + 20, // 14: agentevals.api.v1alpha1.TraceResult.performance_metrics:type_name -> google.protobuf.Struct + 12, // 15: agentevals.api.v1alpha1.TraceResult.turn_refs:type_name -> agentevals.api.v1alpha1.TurnRef + 13, // 16: agentevals.api.v1alpha1.RunResult.trace_results:type_name -> agentevals.api.v1alpha1.TraceResult + 20, // 17: agentevals.api.v1alpha1.RunResult.performance_metrics:type_name -> google.protobuf.Struct + 22, // 18: agentevals.api.v1alpha1.EvaluateTracesRequest.traces:type_name -> agentevals.api.v1alpha1.TraceInput + 20, // 19: agentevals.api.v1alpha1.EvaluateTracesRequest.adk_eval_set:type_name -> google.protobuf.Struct + 9, // 20: agentevals.api.v1alpha1.EvaluateTracesRequest.params:type_name -> agentevals.api.v1alpha1.EvalParams + 19, // 21: agentevals.api.v1alpha1.EvaluateTracesRequest.credential_refs:type_name -> agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry + 20, // 22: agentevals.api.v1alpha1.GroupPerformance.performance_metrics:type_name -> google.protobuf.Struct + 20, // 23: agentevals.api.v1alpha1.GroupPerformance.trace_metadata:type_name -> google.protobuf.Struct + 16, // 24: agentevals.api.v1alpha1.EvaluateTracesResponse.progress:type_name -> agentevals.api.v1alpha1.EvaluationProgress + 17, // 25: agentevals.api.v1alpha1.EvaluateTracesResponse.performance:type_name -> agentevals.api.v1alpha1.GroupPerformance + 13, // 26: agentevals.api.v1alpha1.EvaluateTracesResponse.trace_result:type_name -> agentevals.api.v1alpha1.TraceResult + 14, // 27: agentevals.api.v1alpha1.EvaluateTracesResponse.done:type_name -> agentevals.api.v1alpha1.RunResult + 20, // 28: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry.value:type_name -> google.protobuf.Struct + 2, // 29: agentevals.api.v1alpha1.EvaluationService.ListMetrics:input_type -> agentevals.api.v1alpha1.ListMetricsRequest + 15, // 30: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:input_type -> agentevals.api.v1alpha1.EvaluateTracesRequest + 4, // 31: agentevals.api.v1alpha1.EvaluationService.ListMetrics:output_type -> agentevals.api.v1alpha1.ListMetricsResponse + 18, // 32: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:output_type -> agentevals.api.v1alpha1.EvaluateTracesResponse + 31, // [31:33] is the sub-list for method output_type + 29, // [29:31] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_evaluation_proto_init() } +func file_agentevals_api_v1alpha1_evaluation_proto_init() { + if File_agentevals_api_v1alpha1_evaluation_proto != nil { + return + } + file_agentevals_api_v1alpha1_common_proto_init() + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[3].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6].OneofWrappers = []any{ + (*Evaluator_Builtin)(nil), + (*Evaluator_Code)(nil), + (*Evaluator_Remote)(nil), + } + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13].OneofWrappers = []any{ + (*EvaluateTracesRequest_AdkEvalSet)(nil), + } + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16].OneofWrappers = []any{ + (*EvaluateTracesResponse_Progress)(nil), + (*EvaluateTracesResponse_Performance)(nil), + (*EvaluateTracesResponse_TraceResult)(nil), + (*EvaluateTracesResponse_Done)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc), len(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc)), + NumEnums: 2, + NumMessages: 18, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agentevals_api_v1alpha1_evaluation_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_evaluation_proto_depIdxs, + EnumInfos: file_agentevals_api_v1alpha1_evaluation_proto_enumTypes, + MessageInfos: file_agentevals_api_v1alpha1_evaluation_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_evaluation_proto = out.File + file_agentevals_api_v1alpha1_evaluation_proto_goTypes = nil + file_agentevals_api_v1alpha1_evaluation_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/evaluation_grpc.pb.go b/gen/go/agentevals/api/v1alpha1/evaluation_grpc.pb.go new file mode 100644 index 00000000..58675194 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/evaluation_grpc.pb.go @@ -0,0 +1,169 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: agentevals/api/v1alpha1/evaluation.proto + +package apiv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + EvaluationService_ListMetrics_FullMethodName = "/agentevals.api.v1alpha1.EvaluationService/ListMetrics" + EvaluationService_EvaluateTraces_FullMethodName = "/agentevals.api.v1alpha1.EvaluationService/EvaluateTraces" +) + +// EvaluationServiceClient is the client API for EvaluationService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EvaluationServiceClient interface { + ListMetrics(ctx context.Context, in *ListMetricsRequest, opts ...grpc.CallOption) (*ListMetricsResponse, error) + // Evaluates traces. Streams progress, each group's performance metrics before evaluation, each + // group's results, and the final result, after which the stream ends. When run storage is + // configured, the evaluation is also recorded as a run. + EvaluateTraces(ctx context.Context, in *EvaluateTracesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[EvaluateTracesResponse], error) +} + +type evaluationServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewEvaluationServiceClient(cc grpc.ClientConnInterface) EvaluationServiceClient { + return &evaluationServiceClient{cc} +} + +func (c *evaluationServiceClient) ListMetrics(ctx context.Context, in *ListMetricsRequest, opts ...grpc.CallOption) (*ListMetricsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListMetricsResponse) + err := c.cc.Invoke(ctx, EvaluationService_ListMetrics_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *evaluationServiceClient) EvaluateTraces(ctx context.Context, in *EvaluateTracesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[EvaluateTracesResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &EvaluationService_ServiceDesc.Streams[0], EvaluationService_EvaluateTraces_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[EvaluateTracesRequest, EvaluateTracesResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type EvaluationService_EvaluateTracesClient = grpc.ServerStreamingClient[EvaluateTracesResponse] + +// EvaluationServiceServer is the server API for EvaluationService service. +// All implementations must embed UnimplementedEvaluationServiceServer +// for forward compatibility. +type EvaluationServiceServer interface { + ListMetrics(context.Context, *ListMetricsRequest) (*ListMetricsResponse, error) + // Evaluates traces. Streams progress, each group's performance metrics before evaluation, each + // group's results, and the final result, after which the stream ends. When run storage is + // configured, the evaluation is also recorded as a run. + EvaluateTraces(*EvaluateTracesRequest, grpc.ServerStreamingServer[EvaluateTracesResponse]) error + mustEmbedUnimplementedEvaluationServiceServer() +} + +// UnimplementedEvaluationServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedEvaluationServiceServer struct{} + +func (UnimplementedEvaluationServiceServer) ListMetrics(context.Context, *ListMetricsRequest) (*ListMetricsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListMetrics not implemented") +} +func (UnimplementedEvaluationServiceServer) EvaluateTraces(*EvaluateTracesRequest, grpc.ServerStreamingServer[EvaluateTracesResponse]) error { + return status.Error(codes.Unimplemented, "method EvaluateTraces not implemented") +} +func (UnimplementedEvaluationServiceServer) mustEmbedUnimplementedEvaluationServiceServer() {} +func (UnimplementedEvaluationServiceServer) testEmbeddedByValue() {} + +// UnsafeEvaluationServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EvaluationServiceServer will +// result in compilation errors. +type UnsafeEvaluationServiceServer interface { + mustEmbedUnimplementedEvaluationServiceServer() +} + +func RegisterEvaluationServiceServer(s grpc.ServiceRegistrar, srv EvaluationServiceServer) { + // If the following call panics, it indicates UnimplementedEvaluationServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&EvaluationService_ServiceDesc, srv) +} + +func _EvaluationService_ListMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListMetricsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EvaluationServiceServer).ListMetrics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: EvaluationService_ListMetrics_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EvaluationServiceServer).ListMetrics(ctx, req.(*ListMetricsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _EvaluationService_EvaluateTraces_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(EvaluateTracesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(EvaluationServiceServer).EvaluateTraces(m, &grpc.GenericServerStream[EvaluateTracesRequest, EvaluateTracesResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type EvaluationService_EvaluateTracesServer = grpc.ServerStreamingServer[EvaluateTracesResponse] + +// EvaluationService_ServiceDesc is the grpc.ServiceDesc for EvaluationService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var EvaluationService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agentevals.api.v1alpha1.EvaluationService", + HandlerType: (*EvaluationServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ListMetrics", + Handler: _EvaluationService_ListMetrics_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "EvaluateTraces", + Handler: _EvaluationService_EvaluateTraces_Handler, + ServerStreams: true, + }, + }, + Metadata: "agentevals/api/v1alpha1/evaluation.proto", +} diff --git a/gen/go/agentevals/api/v1alpha1/runs.pb.go b/gen/go/agentevals/api/v1alpha1/runs.pb.go new file mode 100644 index 00000000..45929018 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/runs.pb.go @@ -0,0 +1,1794 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/runs.proto + +package apiv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type RunStatus int32 + +const ( + RunStatus_RUN_STATUS_UNSPECIFIED RunStatus = 0 + RunStatus_RUN_STATUS_QUEUED RunStatus = 1 + RunStatus_RUN_STATUS_RUNNING RunStatus = 2 + RunStatus_RUN_STATUS_SUCCEEDED RunStatus = 3 + RunStatus_RUN_STATUS_FAILED RunStatus = 4 + RunStatus_RUN_STATUS_CANCELLED RunStatus = 5 +) + +// Enum value maps for RunStatus. +var ( + RunStatus_name = map[int32]string{ + 0: "RUN_STATUS_UNSPECIFIED", + 1: "RUN_STATUS_QUEUED", + 2: "RUN_STATUS_RUNNING", + 3: "RUN_STATUS_SUCCEEDED", + 4: "RUN_STATUS_FAILED", + 5: "RUN_STATUS_CANCELLED", + } + RunStatus_value = map[string]int32{ + "RUN_STATUS_UNSPECIFIED": 0, + "RUN_STATUS_QUEUED": 1, + "RUN_STATUS_RUNNING": 2, + "RUN_STATUS_SUCCEEDED": 3, + "RUN_STATUS_FAILED": 4, + "RUN_STATUS_CANCELLED": 5, + } +) + +func (x RunStatus) Enum() *RunStatus { + p := new(RunStatus) + *p = x + return p +} + +func (x RunStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RunStatus) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_runs_proto_enumTypes[0].Descriptor() +} + +func (RunStatus) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_runs_proto_enumTypes[0] +} + +func (x RunStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RunStatus.Descriptor instead. +func (RunStatus) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{0} +} + +type ResultStatus int32 + +const ( + ResultStatus_RESULT_STATUS_UNSPECIFIED ResultStatus = 0 + ResultStatus_RESULT_STATUS_PASSED ResultStatus = 1 + ResultStatus_RESULT_STATUS_FAILED ResultStatus = 2 + ResultStatus_RESULT_STATUS_ERRORED ResultStatus = 3 + ResultStatus_RESULT_STATUS_SKIPPED ResultStatus = 4 +) + +// Enum value maps for ResultStatus. +var ( + ResultStatus_name = map[int32]string{ + 0: "RESULT_STATUS_UNSPECIFIED", + 1: "RESULT_STATUS_PASSED", + 2: "RESULT_STATUS_FAILED", + 3: "RESULT_STATUS_ERRORED", + 4: "RESULT_STATUS_SKIPPED", + } + ResultStatus_value = map[string]int32{ + "RESULT_STATUS_UNSPECIFIED": 0, + "RESULT_STATUS_PASSED": 1, + "RESULT_STATUS_FAILED": 2, + "RESULT_STATUS_ERRORED": 3, + "RESULT_STATUS_SKIPPED": 4, + } +) + +func (x ResultStatus) Enum() *ResultStatus { + p := new(ResultStatus) + *p = x + return p +} + +func (x ResultStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ResultStatus) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_runs_proto_enumTypes[1].Descriptor() +} + +func (ResultStatus) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_runs_proto_enumTypes[1] +} + +func (x ResultStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ResultStatus.Descriptor instead. +func (ResultStatus) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{1} +} + +type EvaluatorType int32 + +const ( + EvaluatorType_EVALUATOR_TYPE_UNSPECIFIED EvaluatorType = 0 + EvaluatorType_EVALUATOR_TYPE_BUILTIN EvaluatorType = 1 + EvaluatorType_EVALUATOR_TYPE_CODE EvaluatorType = 2 + EvaluatorType_EVALUATOR_TYPE_REMOTE EvaluatorType = 3 +) + +// Enum value maps for EvaluatorType. +var ( + EvaluatorType_name = map[int32]string{ + 0: "EVALUATOR_TYPE_UNSPECIFIED", + 1: "EVALUATOR_TYPE_BUILTIN", + 2: "EVALUATOR_TYPE_CODE", + 3: "EVALUATOR_TYPE_REMOTE", + } + EvaluatorType_value = map[string]int32{ + "EVALUATOR_TYPE_UNSPECIFIED": 0, + "EVALUATOR_TYPE_BUILTIN": 1, + "EVALUATOR_TYPE_CODE": 2, + "EVALUATOR_TYPE_REMOTE": 3, + } +) + +func (x EvaluatorType) Enum() *EvaluatorType { + p := new(EvaluatorType) + *p = x + return p +} + +func (x EvaluatorType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EvaluatorType) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_runs_proto_enumTypes[2].Descriptor() +} + +func (EvaluatorType) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_runs_proto_enumTypes[2] +} + +func (x EvaluatorType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EvaluatorType.Descriptor instead. +func (EvaluatorType) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{2} +} + +type RunApproach int32 + +const ( + // RUN_APPROACH_TRACE_REPLAY. + RunApproach_RUN_APPROACH_UNSPECIFIED RunApproach = 0 + // Evaluate recorded traces. + RunApproach_RUN_APPROACH_TRACE_REPLAY RunApproach = 1 +) + +// Enum value maps for RunApproach. +var ( + RunApproach_name = map[int32]string{ + 0: "RUN_APPROACH_UNSPECIFIED", + 1: "RUN_APPROACH_TRACE_REPLAY", + } + RunApproach_value = map[string]int32{ + "RUN_APPROACH_UNSPECIFIED": 0, + "RUN_APPROACH_TRACE_REPLAY": 1, + } +) + +func (x RunApproach) Enum() *RunApproach { + p := new(RunApproach) + *p = x + return p +} + +func (x RunApproach) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RunApproach) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_runs_proto_enumTypes[3].Descriptor() +} + +func (RunApproach) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_runs_proto_enumTypes[3] +} + +func (x RunApproach) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RunApproach.Descriptor instead. +func (RunApproach) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{3} +} + +// A trace fetched with GET base_url/trace_id and the run's context headers. +type HttpTraceTarget struct { + state protoimpl.MessageState `protogen:"open.v1"` + BaseUrl string `protobuf:"bytes,1,opt,name=base_url,json=baseUrl,proto3" json:"base_url,omitempty"` + // 16 or 32 hex characters. + TraceId string `protobuf:"bytes,2,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + // TRACE_FORMAT_OTLP_JSON or TRACE_FORMAT_JAEGER_JSON. + Format TraceFormat `protobuf:"varint,3,opt,name=format,proto3,enum=agentevals.api.v1alpha1.TraceFormat" json:"format,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpTraceTarget) Reset() { + *x = HttpTraceTarget{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpTraceTarget) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpTraceTarget) ProtoMessage() {} + +func (x *HttpTraceTarget) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpTraceTarget.ProtoReflect.Descriptor instead. +func (*HttpTraceTarget) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{0} +} + +func (x *HttpTraceTarget) GetBaseUrl() string { + if x != nil { + return x.BaseUrl + } + return "" +} + +func (x *HttpTraceTarget) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *HttpTraceTarget) GetFormat() TraceFormat { + if x != nil { + return x.Format + } + return TraceFormat_TRACE_FORMAT_UNSPECIFIED +} + +// A run recorded from a synchronous upload. The traces are not kept, so it cannot run again. +type UploadedTraces struct { + state protoimpl.MessageState `protogen:"open.v1"` + TraceCount int32 `protobuf:"varint,1,opt,name=trace_count,json=traceCount,proto3" json:"trace_count,omitempty"` + TraceFiles []string `protobuf:"bytes,2,rep,name=trace_files,json=traceFiles,proto3" json:"trace_files,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadedTraces) Reset() { + *x = UploadedTraces{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadedTraces) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadedTraces) ProtoMessage() {} + +func (x *UploadedTraces) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadedTraces.ProtoReflect.Descriptor instead. +func (*UploadedTraces) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{1} +} + +func (x *UploadedTraces) GetTraceCount() int32 { + if x != nil { + return x.TraceCount + } + return 0 +} + +func (x *UploadedTraces) GetTraceFiles() []string { + if x != nil { + return x.TraceFiles + } + return nil +} + +type TraceTarget struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: + // + // *TraceTarget_Inline + // *TraceTarget_Otlp + // *TraceTarget_Http + // *TraceTarget_Uploaded + Kind isTraceTarget_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TraceTarget) Reset() { + *x = TraceTarget{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceTarget) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceTarget) ProtoMessage() {} + +func (x *TraceTarget) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceTarget.ProtoReflect.Descriptor instead. +func (*TraceTarget) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{2} +} + +func (x *TraceTarget) GetKind() isTraceTarget_Kind { + if x != nil { + return x.Kind + } + return nil +} + +func (x *TraceTarget) GetInline() *TraceDocument { + if x != nil { + if x, ok := x.Kind.(*TraceTarget_Inline); ok { + return x.Inline + } + } + return nil +} + +func (x *TraceTarget) GetOtlp() *OtlpTelemetry { + if x != nil { + if x, ok := x.Kind.(*TraceTarget_Otlp); ok { + return x.Otlp + } + } + return nil +} + +func (x *TraceTarget) GetHttp() *HttpTraceTarget { + if x != nil { + if x, ok := x.Kind.(*TraceTarget_Http); ok { + return x.Http + } + } + return nil +} + +func (x *TraceTarget) GetUploaded() *UploadedTraces { + if x != nil { + if x, ok := x.Kind.(*TraceTarget_Uploaded); ok { + return x.Uploaded + } + } + return nil +} + +type isTraceTarget_Kind interface { + isTraceTarget_Kind() +} + +type TraceTarget_Inline struct { + // A trace document in the spec, OTLP/JSON or Jaeger JSON. + Inline *TraceDocument `protobuf:"bytes,1,opt,name=inline,proto3,oneof"` +} + +type TraceTarget_Otlp struct { + // OTLP protobuf. Stored, and returned, as an inline OTLP/JSON document. + Otlp *OtlpTelemetry `protobuf:"bytes,2,opt,name=otlp,proto3,oneof"` +} + +type TraceTarget_Http struct { + Http *HttpTraceTarget `protobuf:"bytes,3,opt,name=http,proto3,oneof"` +} + +type TraceTarget_Uploaded struct { + // Output only. + Uploaded *UploadedTraces `protobuf:"bytes,4,opt,name=uploaded,proto3,oneof"` +} + +func (*TraceTarget_Inline) isTraceTarget_Kind() {} + +func (*TraceTarget_Otlp) isTraceTarget_Kind() {} + +func (*TraceTarget_Http) isTraceTarget_Kind() {} + +func (*TraceTarget_Uploaded) isTraceTarget_Kind() {} + +type RunContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Headers for fetching an http target. Values are returned redacted. + Headers map[string]string `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunContext) Reset() { + *x = RunContext{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunContext) ProtoMessage() {} + +func (x *RunContext) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunContext.ProtoReflect.Descriptor instead. +func (*RunContext) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{3} +} + +func (x *RunContext) GetHeaders() map[string]string { + if x != nil { + return x.Headers + } + return nil +} + +type RunSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + Approach RunApproach `protobuf:"varint,1,opt,name=approach,proto3,enum=agentevals.api.v1alpha1.RunApproach" json:"approach,omitempty"` + Target *TraceTarget `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` + // The golden. + // + // Types that are valid to be assigned to EvalSet: + // + // *RunSpec_AdkEvalSet + EvalSet isRunSpec_EvalSet `protobuf_oneof:"eval_set"` + EvalConfig *EvalParams `protobuf:"bytes,4,opt,name=eval_config,json=evalConfig,proto3" json:"eval_config,omitempty"` + // Result sinks, for example {"kind": "http_webhook", "url": "...", "headers": {...}}. Header + // values are returned redacted. + Sinks []*structpb.Struct `protobuf:"bytes,5,rep,name=sinks,proto3" json:"sinks,omitempty"` + Context *RunContext `protobuf:"bytes,6,opt,name=context,proto3" json:"context,omitempty"` + // Logical credential name to a secret reference, as in EvaluateTracesRequest. + CredentialRefs map[string]*structpb.Struct `protobuf:"bytes,7,rep,name=credential_refs,json=credentialRefs,proto3" json:"credential_refs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunSpec) Reset() { + *x = RunSpec{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunSpec) ProtoMessage() {} + +func (x *RunSpec) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunSpec.ProtoReflect.Descriptor instead. +func (*RunSpec) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{4} +} + +func (x *RunSpec) GetApproach() RunApproach { + if x != nil { + return x.Approach + } + return RunApproach_RUN_APPROACH_UNSPECIFIED +} + +func (x *RunSpec) GetTarget() *TraceTarget { + if x != nil { + return x.Target + } + return nil +} + +func (x *RunSpec) GetEvalSet() isRunSpec_EvalSet { + if x != nil { + return x.EvalSet + } + return nil +} + +func (x *RunSpec) GetAdkEvalSet() *structpb.Struct { + if x != nil { + if x, ok := x.EvalSet.(*RunSpec_AdkEvalSet); ok { + return x.AdkEvalSet + } + } + return nil +} + +func (x *RunSpec) GetEvalConfig() *EvalParams { + if x != nil { + return x.EvalConfig + } + return nil +} + +func (x *RunSpec) GetSinks() []*structpb.Struct { + if x != nil { + return x.Sinks + } + return nil +} + +func (x *RunSpec) GetContext() *RunContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *RunSpec) GetCredentialRefs() map[string]*structpb.Struct { + if x != nil { + return x.CredentialRefs + } + return nil +} + +type isRunSpec_EvalSet interface { + isRunSpec_EvalSet() +} + +type RunSpec_AdkEvalSet struct { + // An ADK EvalSet as JSON. + AdkEvalSet *structpb.Struct `protobuf:"bytes,3,opt,name=adk_eval_set,json=adkEvalSet,proto3,oneof"` +} + +func (*RunSpec_AdkEvalSet) isRunSpec_EvalSet() {} + +type Run struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A UUID. + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + Status RunStatus `protobuf:"varint,2,opt,name=status,proto3,enum=agentevals.api.v1alpha1.RunStatus" json:"status,omitempty"` + Spec *RunSpec `protobuf:"bytes,3,opt,name=spec,proto3" json:"spec,omitempty"` + Attempt int32 `protobuf:"varint,4,opt,name=attempt,proto3" json:"attempt,omitempty"` + WorkerId string `protobuf:"bytes,5,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty"` + Error string `protobuf:"bytes,6,opt,name=error,proto3" json:"error,omitempty"` + Summary *structpb.Struct `protobuf:"bytes,7,opt,name=summary,proto3" json:"summary,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + FinishedAt *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=finished_at,json=finishedAt,proto3" json:"finished_at,omitempty"` + CancelRequested bool `protobuf:"varint,11,opt,name=cancel_requested,json=cancelRequested,proto3" json:"cancel_requested,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Run) Reset() { + *x = Run{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Run) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Run) ProtoMessage() {} + +func (x *Run) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Run.ProtoReflect.Descriptor instead. +func (*Run) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{5} +} + +func (x *Run) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *Run) GetStatus() RunStatus { + if x != nil { + return x.Status + } + return RunStatus_RUN_STATUS_UNSPECIFIED +} + +func (x *Run) GetSpec() *RunSpec { + if x != nil { + return x.Spec + } + return nil +} + +func (x *Run) GetAttempt() int32 { + if x != nil { + return x.Attempt + } + return 0 +} + +func (x *Run) GetWorkerId() string { + if x != nil { + return x.WorkerId + } + return "" +} + +func (x *Run) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Run) GetSummary() *structpb.Struct { + if x != nil { + return x.Summary + } + return nil +} + +func (x *Run) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *Run) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *Run) GetFinishedAt() *timestamppb.Timestamp { + if x != nil { + return x.FinishedAt + } + return nil +} + +func (x *Run) GetCancelRequested() bool { + if x != nil { + return x.CancelRequested + } + return false +} + +// One evaluator's result for one eval set item. +type Result struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Stable: a SHA-256 of run id, item id and evaluator name. + ResultId string `protobuf:"bytes,1,opt,name=result_id,json=resultId,proto3" json:"result_id,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + EvalSetItemId string `protobuf:"bytes,3,opt,name=eval_set_item_id,json=evalSetItemId,proto3" json:"eval_set_item_id,omitempty"` + EvalSetItemName string `protobuf:"bytes,4,opt,name=eval_set_item_name,json=evalSetItemName,proto3" json:"eval_set_item_name,omitempty"` + EvaluatorName string `protobuf:"bytes,5,opt,name=evaluator_name,json=evaluatorName,proto3" json:"evaluator_name,omitempty"` + EvaluatorType EvaluatorType `protobuf:"varint,6,opt,name=evaluator_type,json=evaluatorType,proto3,enum=agentevals.api.v1alpha1.EvaluatorType" json:"evaluator_type,omitempty"` + Status ResultStatus `protobuf:"varint,7,opt,name=status,proto3,enum=agentevals.api.v1alpha1.ResultStatus" json:"status,omitempty"` + Score *float64 `protobuf:"fixed64,8,opt,name=score,proto3,oneof" json:"score,omitempty"` + // One entry per turn, in turn order; status is not recorded. + PerInvocation []*InvocationResult `protobuf:"bytes,9,rep,name=per_invocation,json=perInvocation,proto3" json:"per_invocation,omitempty"` + TraceId string `protobuf:"bytes,10,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanId string `protobuf:"bytes,11,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + Details *structpb.Struct `protobuf:"bytes,12,opt,name=details,proto3" json:"details,omitempty"` + ErrorText string `protobuf:"bytes,13,opt,name=error_text,json=errorText,proto3" json:"error_text,omitempty"` + TokensUsed *structpb.Struct `protobuf:"bytes,14,opt,name=tokens_used,json=tokensUsed,proto3" json:"tokens_used,omitempty"` + LatencyMs *int64 `protobuf:"varint,15,opt,name=latency_ms,json=latencyMs,proto3,oneof" json:"latency_ms,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,16,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Result) Reset() { + *x = Result{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Result) ProtoMessage() {} + +func (x *Result) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Result.ProtoReflect.Descriptor instead. +func (*Result) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{6} +} + +func (x *Result) GetResultId() string { + if x != nil { + return x.ResultId + } + return "" +} + +func (x *Result) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *Result) GetEvalSetItemId() string { + if x != nil { + return x.EvalSetItemId + } + return "" +} + +func (x *Result) GetEvalSetItemName() string { + if x != nil { + return x.EvalSetItemName + } + return "" +} + +func (x *Result) GetEvaluatorName() string { + if x != nil { + return x.EvaluatorName + } + return "" +} + +func (x *Result) GetEvaluatorType() EvaluatorType { + if x != nil { + return x.EvaluatorType + } + return EvaluatorType_EVALUATOR_TYPE_UNSPECIFIED +} + +func (x *Result) GetStatus() ResultStatus { + if x != nil { + return x.Status + } + return ResultStatus_RESULT_STATUS_UNSPECIFIED +} + +func (x *Result) GetScore() float64 { + if x != nil && x.Score != nil { + return *x.Score + } + return 0 +} + +func (x *Result) GetPerInvocation() []*InvocationResult { + if x != nil { + return x.PerInvocation + } + return nil +} + +func (x *Result) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *Result) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +func (x *Result) GetDetails() *structpb.Struct { + if x != nil { + return x.Details + } + return nil +} + +func (x *Result) GetErrorText() string { + if x != nil { + return x.ErrorText + } + return "" +} + +func (x *Result) GetTokensUsed() *structpb.Struct { + if x != nil { + return x.TokensUsed + } + return nil +} + +func (x *Result) GetLatencyMs() int64 { + if x != nil && x.LatencyMs != nil { + return *x.LatencyMs + } + return 0 +} + +func (x *Result) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +type SubmitRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A UUID chosen by the caller, for idempotent retries. Empty means the server picks one. + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + Spec *RunSpec `protobuf:"bytes,2,opt,name=spec,proto3" json:"spec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitRunRequest) Reset() { + *x = SubmitRunRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitRunRequest) ProtoMessage() {} + +func (x *SubmitRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitRunRequest.ProtoReflect.Descriptor instead. +func (*SubmitRunRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{7} +} + +func (x *SubmitRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *SubmitRunRequest) GetSpec() *RunSpec { + if x != nil { + return x.Spec + } + return nil +} + +type SubmitRunResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Run *Run `protobuf:"bytes,1,opt,name=run,proto3" json:"run,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitRunResponse) Reset() { + *x = SubmitRunResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitRunResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitRunResponse) ProtoMessage() {} + +func (x *SubmitRunResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitRunResponse.ProtoReflect.Descriptor instead. +func (*SubmitRunResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{8} +} + +func (x *SubmitRunResponse) GetRun() *Run { + if x != nil { + return x.Run + } + return nil +} + +type GetRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetRunRequest) Reset() { + *x = GetRunRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetRunRequest) ProtoMessage() {} + +func (x *GetRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetRunRequest.ProtoReflect.Descriptor instead. +func (*GetRunRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{9} +} + +func (x *GetRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type GetRunResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Run *Run `protobuf:"bytes,1,opt,name=run,proto3" json:"run,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetRunResponse) Reset() { + *x = GetRunResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetRunResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetRunResponse) ProtoMessage() {} + +func (x *GetRunResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetRunResponse.ProtoReflect.Descriptor instead. +func (*GetRunResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{10} +} + +func (x *GetRunResponse) GetRun() *Run { + if x != nil { + return x.Run + } + return nil +} + +type ListRunsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Empty means every status. + Statuses []RunStatus `protobuf:"varint,1,rep,packed,name=statuses,proto3,enum=agentevals.api.v1alpha1.RunStatus" json:"statuses,omitempty"` + Page *PageRequest `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListRunsRequest) Reset() { + *x = ListRunsRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListRunsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListRunsRequest) ProtoMessage() {} + +func (x *ListRunsRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListRunsRequest.ProtoReflect.Descriptor instead. +func (*ListRunsRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{11} +} + +func (x *ListRunsRequest) GetStatuses() []RunStatus { + if x != nil { + return x.Statuses + } + return nil +} + +func (x *ListRunsRequest) GetPage() *PageRequest { + if x != nil { + return x.Page + } + return nil +} + +type ListRunsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Runs []*Run `protobuf:"bytes,1,rep,name=runs,proto3" json:"runs,omitempty"` + Page *PageResponse `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListRunsResponse) Reset() { + *x = ListRunsResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListRunsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListRunsResponse) ProtoMessage() {} + +func (x *ListRunsResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListRunsResponse.ProtoReflect.Descriptor instead. +func (*ListRunsResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{12} +} + +func (x *ListRunsResponse) GetRuns() []*Run { + if x != nil { + return x.Runs + } + return nil +} + +func (x *ListRunsResponse) GetPage() *PageResponse { + if x != nil { + return x.Page + } + return nil +} + +type CancelRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CancelRunRequest) Reset() { + *x = CancelRunRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CancelRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelRunRequest) ProtoMessage() {} + +func (x *CancelRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelRunRequest.ProtoReflect.Descriptor instead. +func (*CancelRunRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{13} +} + +func (x *CancelRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type CancelRunResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Run *Run `protobuf:"bytes,1,opt,name=run,proto3" json:"run,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CancelRunResponse) Reset() { + *x = CancelRunResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CancelRunResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelRunResponse) ProtoMessage() {} + +func (x *CancelRunResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelRunResponse.ProtoReflect.Descriptor instead. +func (*CancelRunResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{14} +} + +func (x *CancelRunResponse) GetRun() *Run { + if x != nil { + return x.Run + } + return nil +} + +type ListRunResultsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + Page *PageRequest `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListRunResultsRequest) Reset() { + *x = ListRunResultsRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListRunResultsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListRunResultsRequest) ProtoMessage() {} + +func (x *ListRunResultsRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListRunResultsRequest.ProtoReflect.Descriptor instead. +func (*ListRunResultsRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{15} +} + +func (x *ListRunResultsRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *ListRunResultsRequest) GetPage() *PageRequest { + if x != nil { + return x.Page + } + return nil +} + +type ListRunResultsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Page *PageResponse `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListRunResultsResponse) Reset() { + *x = ListRunResultsResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListRunResultsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListRunResultsResponse) ProtoMessage() {} + +func (x *ListRunResultsResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListRunResultsResponse.ProtoReflect.Descriptor instead. +func (*ListRunResultsResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{16} +} + +func (x *ListRunResultsResponse) GetResults() []*Result { + if x != nil { + return x.Results + } + return nil +} + +func (x *ListRunResultsResponse) GetPage() *PageResponse { + if x != nil { + return x.Page + } + return nil +} + +type WatchRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchRunRequest) Reset() { + *x = WatchRunRequest{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchRunRequest) ProtoMessage() {} + +func (x *WatchRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchRunRequest.ProtoReflect.Descriptor instead. +func (*WatchRunRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{17} +} + +func (x *WatchRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type WatchRunResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The run, sent first and after every change. + Run *Run `protobuf:"bytes,1,opt,name=run,proto3" json:"run,omitempty"` + // Results that are new or changed since the previous message. + Results []*Result `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchRunResponse) Reset() { + *x = WatchRunResponse{} + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchRunResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchRunResponse) ProtoMessage() {} + +func (x *WatchRunResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_runs_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchRunResponse.ProtoReflect.Descriptor instead. +func (*WatchRunResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP(), []int{18} +} + +func (x *WatchRunResponse) GetRun() *Run { + if x != nil { + return x.Run + } + return nil +} + +func (x *WatchRunResponse) GetResults() []*Result { + if x != nil { + return x.Results + } + return nil +} + +var File_agentevals_api_v1alpha1_runs_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_runs_proto_rawDesc = "" + + "\n" + + "\"agentevals/api/v1alpha1/runs.proto\x12\x17agentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a(agentevals/api/v1alpha1/evaluation.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x85\x01\n" + + "\x0fHttpTraceTarget\x12\x19\n" + + "\bbase_url\x18\x01 \x01(\tR\abaseUrl\x12\x19\n" + + "\btrace_id\x18\x02 \x01(\tR\atraceId\x12<\n" + + "\x06format\x18\x03 \x01(\x0e2$.agentevals.api.v1alpha1.TraceFormatR\x06format\"R\n" + + "\x0eUploadedTraces\x12\x1f\n" + + "\vtrace_count\x18\x01 \x01(\x05R\n" + + "traceCount\x12\x1f\n" + + "\vtrace_files\x18\x02 \x03(\tR\n" + + "traceFiles\"\x9c\x02\n" + + "\vTraceTarget\x12@\n" + + "\x06inline\x18\x01 \x01(\v2&.agentevals.api.v1alpha1.TraceDocumentH\x00R\x06inline\x12<\n" + + "\x04otlp\x18\x02 \x01(\v2&.agentevals.api.v1alpha1.OtlpTelemetryH\x00R\x04otlp\x12>\n" + + "\x04http\x18\x03 \x01(\v2(.agentevals.api.v1alpha1.HttpTraceTargetH\x00R\x04http\x12E\n" + + "\buploaded\x18\x04 \x01(\v2'.agentevals.api.v1alpha1.UploadedTracesH\x00R\buploadedB\x06\n" + + "\x04kind\"\x94\x01\n" + + "\n" + + "RunContext\x12J\n" + + "\aheaders\x18\x01 \x03(\v20.agentevals.api.v1alpha1.RunContext.HeadersEntryR\aheaders\x1a:\n" + + "\fHeadersEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc1\x04\n" + + "\aRunSpec\x12@\n" + + "\bapproach\x18\x01 \x01(\x0e2$.agentevals.api.v1alpha1.RunApproachR\bapproach\x12<\n" + + "\x06target\x18\x02 \x01(\v2$.agentevals.api.v1alpha1.TraceTargetR\x06target\x12;\n" + + "\fadk_eval_set\x18\x03 \x01(\v2\x17.google.protobuf.StructH\x00R\n" + + "adkEvalSet\x12D\n" + + "\veval_config\x18\x04 \x01(\v2#.agentevals.api.v1alpha1.EvalParamsR\n" + + "evalConfig\x12-\n" + + "\x05sinks\x18\x05 \x03(\v2\x17.google.protobuf.StructR\x05sinks\x12=\n" + + "\acontext\x18\x06 \x01(\v2#.agentevals.api.v1alpha1.RunContextR\acontext\x12]\n" + + "\x0fcredential_refs\x18\a \x03(\v24.agentevals.api.v1alpha1.RunSpec.CredentialRefsEntryR\x0ecredentialRefs\x1aZ\n" + + "\x13CredentialRefsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12-\n" + + "\x05value\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x05value:\x028\x01B\n" + + "\n" + + "\beval_set\"\xec\x03\n" + + "\x03Run\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12:\n" + + "\x06status\x18\x02 \x01(\x0e2\".agentevals.api.v1alpha1.RunStatusR\x06status\x124\n" + + "\x04spec\x18\x03 \x01(\v2 .agentevals.api.v1alpha1.RunSpecR\x04spec\x12\x18\n" + + "\aattempt\x18\x04 \x01(\x05R\aattempt\x12\x1b\n" + + "\tworker_id\x18\x05 \x01(\tR\bworkerId\x12\x14\n" + + "\x05error\x18\x06 \x01(\tR\x05error\x121\n" + + "\asummary\x18\a \x01(\v2\x17.google.protobuf.StructR\asummary\x129\n" + + "\n" + + "created_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + + "\n" + + "started_at\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x12;\n" + + "\vfinished_at\x18\n" + + " \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "finishedAt\x12)\n" + + "\x10cancel_requested\x18\v \x01(\bR\x0fcancelRequested\"\xec\x05\n" + + "\x06Result\x12\x1b\n" + + "\tresult_id\x18\x01 \x01(\tR\bresultId\x12\x15\n" + + "\x06run_id\x18\x02 \x01(\tR\x05runId\x12'\n" + + "\x10eval_set_item_id\x18\x03 \x01(\tR\revalSetItemId\x12+\n" + + "\x12eval_set_item_name\x18\x04 \x01(\tR\x0fevalSetItemName\x12%\n" + + "\x0eevaluator_name\x18\x05 \x01(\tR\revaluatorName\x12M\n" + + "\x0eevaluator_type\x18\x06 \x01(\x0e2&.agentevals.api.v1alpha1.EvaluatorTypeR\revaluatorType\x12=\n" + + "\x06status\x18\a \x01(\x0e2%.agentevals.api.v1alpha1.ResultStatusR\x06status\x12\x19\n" + + "\x05score\x18\b \x01(\x01H\x00R\x05score\x88\x01\x01\x12P\n" + + "\x0eper_invocation\x18\t \x03(\v2).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n" + + "\btrace_id\x18\n" + + " \x01(\tR\atraceId\x12\x17\n" + + "\aspan_id\x18\v \x01(\tR\x06spanId\x121\n" + + "\adetails\x18\f \x01(\v2\x17.google.protobuf.StructR\adetails\x12\x1d\n" + + "\n" + + "error_text\x18\r \x01(\tR\terrorText\x128\n" + + "\vtokens_used\x18\x0e \x01(\v2\x17.google.protobuf.StructR\n" + + "tokensUsed\x12\"\n" + + "\n" + + "latency_ms\x18\x0f \x01(\x03H\x01R\tlatencyMs\x88\x01\x01\x129\n" + + "\n" + + "created_at\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAtB\b\n" + + "\x06_scoreB\r\n" + + "\v_latency_ms\"_\n" + + "\x10SubmitRunRequest\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\x124\n" + + "\x04spec\x18\x02 \x01(\v2 .agentevals.api.v1alpha1.RunSpecR\x04spec\"C\n" + + "\x11SubmitRunResponse\x12.\n" + + "\x03run\x18\x01 \x01(\v2\x1c.agentevals.api.v1alpha1.RunR\x03run\"&\n" + + "\rGetRunRequest\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\"@\n" + + "\x0eGetRunResponse\x12.\n" + + "\x03run\x18\x01 \x01(\v2\x1c.agentevals.api.v1alpha1.RunR\x03run\"\x8b\x01\n" + + "\x0fListRunsRequest\x12>\n" + + "\bstatuses\x18\x01 \x03(\x0e2\".agentevals.api.v1alpha1.RunStatusR\bstatuses\x128\n" + + "\x04page\x18\x02 \x01(\v2$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x7f\n" + + "\x10ListRunsResponse\x120\n" + + "\x04runs\x18\x01 \x03(\v2\x1c.agentevals.api.v1alpha1.RunR\x04runs\x129\n" + + "\x04page\x18\x02 \x01(\v2%.agentevals.api.v1alpha1.PageResponseR\x04page\")\n" + + "\x10CancelRunRequest\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\"C\n" + + "\x11CancelRunResponse\x12.\n" + + "\x03run\x18\x01 \x01(\v2\x1c.agentevals.api.v1alpha1.RunR\x03run\"h\n" + + "\x15ListRunResultsRequest\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\x128\n" + + "\x04page\x18\x02 \x01(\v2$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x8e\x01\n" + + "\x16ListRunResultsResponse\x129\n" + + "\aresults\x18\x01 \x03(\v2\x1f.agentevals.api.v1alpha1.ResultR\aresults\x129\n" + + "\x04page\x18\x02 \x01(\v2%.agentevals.api.v1alpha1.PageResponseR\x04page\"(\n" + + "\x0fWatchRunRequest\x12\x15\n" + + "\x06run_id\x18\x01 \x01(\tR\x05runId\"}\n" + + "\x10WatchRunResponse\x12.\n" + + "\x03run\x18\x01 \x01(\v2\x1c.agentevals.api.v1alpha1.RunR\x03run\x129\n" + + "\aresults\x18\x02 \x03(\v2\x1f.agentevals.api.v1alpha1.ResultR\aresults*\xa1\x01\n" + + "\tRunStatus\x12\x1a\n" + + "\x16RUN_STATUS_UNSPECIFIED\x10\x00\x12\x15\n" + + "\x11RUN_STATUS_QUEUED\x10\x01\x12\x16\n" + + "\x12RUN_STATUS_RUNNING\x10\x02\x12\x18\n" + + "\x14RUN_STATUS_SUCCEEDED\x10\x03\x12\x15\n" + + "\x11RUN_STATUS_FAILED\x10\x04\x12\x18\n" + + "\x14RUN_STATUS_CANCELLED\x10\x05*\x97\x01\n" + + "\fResultStatus\x12\x1d\n" + + "\x19RESULT_STATUS_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14RESULT_STATUS_PASSED\x10\x01\x12\x18\n" + + "\x14RESULT_STATUS_FAILED\x10\x02\x12\x19\n" + + "\x15RESULT_STATUS_ERRORED\x10\x03\x12\x19\n" + + "\x15RESULT_STATUS_SKIPPED\x10\x04*\x7f\n" + + "\rEvaluatorType\x12\x1e\n" + + "\x1aEVALUATOR_TYPE_UNSPECIFIED\x10\x00\x12\x1a\n" + + "\x16EVALUATOR_TYPE_BUILTIN\x10\x01\x12\x17\n" + + "\x13EVALUATOR_TYPE_CODE\x10\x02\x12\x19\n" + + "\x15EVALUATOR_TYPE_REMOTE\x10\x03*J\n" + + "\vRunApproach\x12\x1c\n" + + "\x18RUN_APPROACH_UNSPECIFIED\x10\x00\x12\x1d\n" + + "\x19RUN_APPROACH_TRACE_REPLAY\x10\x012\xe6\x04\n" + + "\n" + + "RunService\x12b\n" + + "\tSubmitRun\x12).agentevals.api.v1alpha1.SubmitRunRequest\x1a*.agentevals.api.v1alpha1.SubmitRunResponse\x12Y\n" + + "\x06GetRun\x12&.agentevals.api.v1alpha1.GetRunRequest\x1a'.agentevals.api.v1alpha1.GetRunResponse\x12_\n" + + "\bListRuns\x12(.agentevals.api.v1alpha1.ListRunsRequest\x1a).agentevals.api.v1alpha1.ListRunsResponse\x12b\n" + + "\tCancelRun\x12).agentevals.api.v1alpha1.CancelRunRequest\x1a*.agentevals.api.v1alpha1.CancelRunResponse\x12q\n" + + "\x0eListRunResults\x12..agentevals.api.v1alpha1.ListRunResultsRequest\x1a/.agentevals.api.v1alpha1.ListRunResultsResponse\x12a\n" + + "\bWatchRun\x12(.agentevals.api.v1alpha1.WatchRunRequest\x1a).agentevals.api.v1alpha1.WatchRunResponse0\x01BQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_runs_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_runs_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_runs_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_runs_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_runs_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_runs_proto_rawDesc), len(file_agentevals_api_v1alpha1_runs_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_runs_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_runs_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_agentevals_api_v1alpha1_runs_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_agentevals_api_v1alpha1_runs_proto_goTypes = []any{ + (RunStatus)(0), // 0: agentevals.api.v1alpha1.RunStatus + (ResultStatus)(0), // 1: agentevals.api.v1alpha1.ResultStatus + (EvaluatorType)(0), // 2: agentevals.api.v1alpha1.EvaluatorType + (RunApproach)(0), // 3: agentevals.api.v1alpha1.RunApproach + (*HttpTraceTarget)(nil), // 4: agentevals.api.v1alpha1.HttpTraceTarget + (*UploadedTraces)(nil), // 5: agentevals.api.v1alpha1.UploadedTraces + (*TraceTarget)(nil), // 6: agentevals.api.v1alpha1.TraceTarget + (*RunContext)(nil), // 7: agentevals.api.v1alpha1.RunContext + (*RunSpec)(nil), // 8: agentevals.api.v1alpha1.RunSpec + (*Run)(nil), // 9: agentevals.api.v1alpha1.Run + (*Result)(nil), // 10: agentevals.api.v1alpha1.Result + (*SubmitRunRequest)(nil), // 11: agentevals.api.v1alpha1.SubmitRunRequest + (*SubmitRunResponse)(nil), // 12: agentevals.api.v1alpha1.SubmitRunResponse + (*GetRunRequest)(nil), // 13: agentevals.api.v1alpha1.GetRunRequest + (*GetRunResponse)(nil), // 14: agentevals.api.v1alpha1.GetRunResponse + (*ListRunsRequest)(nil), // 15: agentevals.api.v1alpha1.ListRunsRequest + (*ListRunsResponse)(nil), // 16: agentevals.api.v1alpha1.ListRunsResponse + (*CancelRunRequest)(nil), // 17: agentevals.api.v1alpha1.CancelRunRequest + (*CancelRunResponse)(nil), // 18: agentevals.api.v1alpha1.CancelRunResponse + (*ListRunResultsRequest)(nil), // 19: agentevals.api.v1alpha1.ListRunResultsRequest + (*ListRunResultsResponse)(nil), // 20: agentevals.api.v1alpha1.ListRunResultsResponse + (*WatchRunRequest)(nil), // 21: agentevals.api.v1alpha1.WatchRunRequest + (*WatchRunResponse)(nil), // 22: agentevals.api.v1alpha1.WatchRunResponse + nil, // 23: agentevals.api.v1alpha1.RunContext.HeadersEntry + nil, // 24: agentevals.api.v1alpha1.RunSpec.CredentialRefsEntry + (TraceFormat)(0), // 25: agentevals.api.v1alpha1.TraceFormat + (*TraceDocument)(nil), // 26: agentevals.api.v1alpha1.TraceDocument + (*OtlpTelemetry)(nil), // 27: agentevals.api.v1alpha1.OtlpTelemetry + (*structpb.Struct)(nil), // 28: google.protobuf.Struct + (*EvalParams)(nil), // 29: agentevals.api.v1alpha1.EvalParams + (*timestamppb.Timestamp)(nil), // 30: google.protobuf.Timestamp + (*InvocationResult)(nil), // 31: agentevals.api.v1alpha1.InvocationResult + (*PageRequest)(nil), // 32: agentevals.api.v1alpha1.PageRequest + (*PageResponse)(nil), // 33: agentevals.api.v1alpha1.PageResponse +} +var file_agentevals_api_v1alpha1_runs_proto_depIdxs = []int32{ + 25, // 0: agentevals.api.v1alpha1.HttpTraceTarget.format:type_name -> agentevals.api.v1alpha1.TraceFormat + 26, // 1: agentevals.api.v1alpha1.TraceTarget.inline:type_name -> agentevals.api.v1alpha1.TraceDocument + 27, // 2: agentevals.api.v1alpha1.TraceTarget.otlp:type_name -> agentevals.api.v1alpha1.OtlpTelemetry + 4, // 3: agentevals.api.v1alpha1.TraceTarget.http:type_name -> agentevals.api.v1alpha1.HttpTraceTarget + 5, // 4: agentevals.api.v1alpha1.TraceTarget.uploaded:type_name -> agentevals.api.v1alpha1.UploadedTraces + 23, // 5: agentevals.api.v1alpha1.RunContext.headers:type_name -> agentevals.api.v1alpha1.RunContext.HeadersEntry + 3, // 6: agentevals.api.v1alpha1.RunSpec.approach:type_name -> agentevals.api.v1alpha1.RunApproach + 6, // 7: agentevals.api.v1alpha1.RunSpec.target:type_name -> agentevals.api.v1alpha1.TraceTarget + 28, // 8: agentevals.api.v1alpha1.RunSpec.adk_eval_set:type_name -> google.protobuf.Struct + 29, // 9: agentevals.api.v1alpha1.RunSpec.eval_config:type_name -> agentevals.api.v1alpha1.EvalParams + 28, // 10: agentevals.api.v1alpha1.RunSpec.sinks:type_name -> google.protobuf.Struct + 7, // 11: agentevals.api.v1alpha1.RunSpec.context:type_name -> agentevals.api.v1alpha1.RunContext + 24, // 12: agentevals.api.v1alpha1.RunSpec.credential_refs:type_name -> agentevals.api.v1alpha1.RunSpec.CredentialRefsEntry + 0, // 13: agentevals.api.v1alpha1.Run.status:type_name -> agentevals.api.v1alpha1.RunStatus + 8, // 14: agentevals.api.v1alpha1.Run.spec:type_name -> agentevals.api.v1alpha1.RunSpec + 28, // 15: agentevals.api.v1alpha1.Run.summary:type_name -> google.protobuf.Struct + 30, // 16: agentevals.api.v1alpha1.Run.created_at:type_name -> google.protobuf.Timestamp + 30, // 17: agentevals.api.v1alpha1.Run.started_at:type_name -> google.protobuf.Timestamp + 30, // 18: agentevals.api.v1alpha1.Run.finished_at:type_name -> google.protobuf.Timestamp + 2, // 19: agentevals.api.v1alpha1.Result.evaluator_type:type_name -> agentevals.api.v1alpha1.EvaluatorType + 1, // 20: agentevals.api.v1alpha1.Result.status:type_name -> agentevals.api.v1alpha1.ResultStatus + 31, // 21: agentevals.api.v1alpha1.Result.per_invocation:type_name -> agentevals.api.v1alpha1.InvocationResult + 28, // 22: agentevals.api.v1alpha1.Result.details:type_name -> google.protobuf.Struct + 28, // 23: agentevals.api.v1alpha1.Result.tokens_used:type_name -> google.protobuf.Struct + 30, // 24: agentevals.api.v1alpha1.Result.created_at:type_name -> google.protobuf.Timestamp + 8, // 25: agentevals.api.v1alpha1.SubmitRunRequest.spec:type_name -> agentevals.api.v1alpha1.RunSpec + 9, // 26: agentevals.api.v1alpha1.SubmitRunResponse.run:type_name -> agentevals.api.v1alpha1.Run + 9, // 27: agentevals.api.v1alpha1.GetRunResponse.run:type_name -> agentevals.api.v1alpha1.Run + 0, // 28: agentevals.api.v1alpha1.ListRunsRequest.statuses:type_name -> agentevals.api.v1alpha1.RunStatus + 32, // 29: agentevals.api.v1alpha1.ListRunsRequest.page:type_name -> agentevals.api.v1alpha1.PageRequest + 9, // 30: agentevals.api.v1alpha1.ListRunsResponse.runs:type_name -> agentevals.api.v1alpha1.Run + 33, // 31: agentevals.api.v1alpha1.ListRunsResponse.page:type_name -> agentevals.api.v1alpha1.PageResponse + 9, // 32: agentevals.api.v1alpha1.CancelRunResponse.run:type_name -> agentevals.api.v1alpha1.Run + 32, // 33: agentevals.api.v1alpha1.ListRunResultsRequest.page:type_name -> agentevals.api.v1alpha1.PageRequest + 10, // 34: agentevals.api.v1alpha1.ListRunResultsResponse.results:type_name -> agentevals.api.v1alpha1.Result + 33, // 35: agentevals.api.v1alpha1.ListRunResultsResponse.page:type_name -> agentevals.api.v1alpha1.PageResponse + 9, // 36: agentevals.api.v1alpha1.WatchRunResponse.run:type_name -> agentevals.api.v1alpha1.Run + 10, // 37: agentevals.api.v1alpha1.WatchRunResponse.results:type_name -> agentevals.api.v1alpha1.Result + 28, // 38: agentevals.api.v1alpha1.RunSpec.CredentialRefsEntry.value:type_name -> google.protobuf.Struct + 11, // 39: agentevals.api.v1alpha1.RunService.SubmitRun:input_type -> agentevals.api.v1alpha1.SubmitRunRequest + 13, // 40: agentevals.api.v1alpha1.RunService.GetRun:input_type -> agentevals.api.v1alpha1.GetRunRequest + 15, // 41: agentevals.api.v1alpha1.RunService.ListRuns:input_type -> agentevals.api.v1alpha1.ListRunsRequest + 17, // 42: agentevals.api.v1alpha1.RunService.CancelRun:input_type -> agentevals.api.v1alpha1.CancelRunRequest + 19, // 43: agentevals.api.v1alpha1.RunService.ListRunResults:input_type -> agentevals.api.v1alpha1.ListRunResultsRequest + 21, // 44: agentevals.api.v1alpha1.RunService.WatchRun:input_type -> agentevals.api.v1alpha1.WatchRunRequest + 12, // 45: agentevals.api.v1alpha1.RunService.SubmitRun:output_type -> agentevals.api.v1alpha1.SubmitRunResponse + 14, // 46: agentevals.api.v1alpha1.RunService.GetRun:output_type -> agentevals.api.v1alpha1.GetRunResponse + 16, // 47: agentevals.api.v1alpha1.RunService.ListRuns:output_type -> agentevals.api.v1alpha1.ListRunsResponse + 18, // 48: agentevals.api.v1alpha1.RunService.CancelRun:output_type -> agentevals.api.v1alpha1.CancelRunResponse + 20, // 49: agentevals.api.v1alpha1.RunService.ListRunResults:output_type -> agentevals.api.v1alpha1.ListRunResultsResponse + 22, // 50: agentevals.api.v1alpha1.RunService.WatchRun:output_type -> agentevals.api.v1alpha1.WatchRunResponse + 45, // [45:51] is the sub-list for method output_type + 39, // [39:45] is the sub-list for method input_type + 39, // [39:39] is the sub-list for extension type_name + 39, // [39:39] is the sub-list for extension extendee + 0, // [0:39] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_runs_proto_init() } +func file_agentevals_api_v1alpha1_runs_proto_init() { + if File_agentevals_api_v1alpha1_runs_proto != nil { + return + } + file_agentevals_api_v1alpha1_common_proto_init() + file_agentevals_api_v1alpha1_evaluation_proto_init() + file_agentevals_api_v1alpha1_runs_proto_msgTypes[2].OneofWrappers = []any{ + (*TraceTarget_Inline)(nil), + (*TraceTarget_Otlp)(nil), + (*TraceTarget_Http)(nil), + (*TraceTarget_Uploaded)(nil), + } + file_agentevals_api_v1alpha1_runs_proto_msgTypes[4].OneofWrappers = []any{ + (*RunSpec_AdkEvalSet)(nil), + } + file_agentevals_api_v1alpha1_runs_proto_msgTypes[6].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_runs_proto_rawDesc), len(file_agentevals_api_v1alpha1_runs_proto_rawDesc)), + NumEnums: 4, + NumMessages: 21, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agentevals_api_v1alpha1_runs_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_runs_proto_depIdxs, + EnumInfos: file_agentevals_api_v1alpha1_runs_proto_enumTypes, + MessageInfos: file_agentevals_api_v1alpha1_runs_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_runs_proto = out.File + file_agentevals_api_v1alpha1_runs_proto_goTypes = nil + file_agentevals_api_v1alpha1_runs_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/runs_grpc.pb.go b/gen/go/agentevals/api/v1alpha1/runs_grpc.pb.go new file mode 100644 index 00000000..d5f82b6c --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/runs_grpc.pb.go @@ -0,0 +1,331 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: agentevals/api/v1alpha1/runs.proto + +package apiv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + RunService_SubmitRun_FullMethodName = "/agentevals.api.v1alpha1.RunService/SubmitRun" + RunService_GetRun_FullMethodName = "/agentevals.api.v1alpha1.RunService/GetRun" + RunService_ListRuns_FullMethodName = "/agentevals.api.v1alpha1.RunService/ListRuns" + RunService_CancelRun_FullMethodName = "/agentevals.api.v1alpha1.RunService/CancelRun" + RunService_ListRunResults_FullMethodName = "/agentevals.api.v1alpha1.RunService/ListRunResults" + RunService_WatchRun_FullMethodName = "/agentevals.api.v1alpha1.RunService/WatchRun" +) + +// RunServiceClient is the client API for RunService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every +// method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. +type RunServiceClient interface { + // Queues a run. Idempotent on run_id: the same id with the same spec returns the existing run; + // a different spec fails with ALREADY_EXISTS and reason RUN_SPEC_CONFLICT. + SubmitRun(ctx context.Context, in *SubmitRunRequest, opts ...grpc.CallOption) (*SubmitRunResponse, error) + GetRun(ctx context.Context, in *GetRunRequest, opts ...grpc.CallOption) (*GetRunResponse, error) + // Newest first. + ListRuns(ctx context.Context, in *ListRunsRequest, opts ...grpc.CallOption) (*ListRunsResponse, error) + // Requests cancellation. Cancelling a finished run changes nothing. + CancelRun(ctx context.Context, in *CancelRunRequest, opts ...grpc.CallOption) (*CancelRunResponse, error) + ListRunResults(ctx context.Context, in *ListRunResultsRequest, opts ...grpc.CallOption) (*ListRunResultsResponse, error) + // The run and its results, then each change, until the run reaches a terminal status. + WatchRun(ctx context.Context, in *WatchRunRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchRunResponse], error) +} + +type runServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewRunServiceClient(cc grpc.ClientConnInterface) RunServiceClient { + return &runServiceClient{cc} +} + +func (c *runServiceClient) SubmitRun(ctx context.Context, in *SubmitRunRequest, opts ...grpc.CallOption) (*SubmitRunResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SubmitRunResponse) + err := c.cc.Invoke(ctx, RunService_SubmitRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *runServiceClient) GetRun(ctx context.Context, in *GetRunRequest, opts ...grpc.CallOption) (*GetRunResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetRunResponse) + err := c.cc.Invoke(ctx, RunService_GetRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *runServiceClient) ListRuns(ctx context.Context, in *ListRunsRequest, opts ...grpc.CallOption) (*ListRunsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListRunsResponse) + err := c.cc.Invoke(ctx, RunService_ListRuns_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *runServiceClient) CancelRun(ctx context.Context, in *CancelRunRequest, opts ...grpc.CallOption) (*CancelRunResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CancelRunResponse) + err := c.cc.Invoke(ctx, RunService_CancelRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *runServiceClient) ListRunResults(ctx context.Context, in *ListRunResultsRequest, opts ...grpc.CallOption) (*ListRunResultsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListRunResultsResponse) + err := c.cc.Invoke(ctx, RunService_ListRunResults_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *runServiceClient) WatchRun(ctx context.Context, in *WatchRunRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchRunResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &RunService_ServiceDesc.Streams[0], RunService_WatchRun_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[WatchRunRequest, WatchRunResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RunService_WatchRunClient = grpc.ServerStreamingClient[WatchRunResponse] + +// RunServiceServer is the server API for RunService service. +// All implementations must embed UnimplementedRunServiceServer +// for forward compatibility. +// +// Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every +// method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. +type RunServiceServer interface { + // Queues a run. Idempotent on run_id: the same id with the same spec returns the existing run; + // a different spec fails with ALREADY_EXISTS and reason RUN_SPEC_CONFLICT. + SubmitRun(context.Context, *SubmitRunRequest) (*SubmitRunResponse, error) + GetRun(context.Context, *GetRunRequest) (*GetRunResponse, error) + // Newest first. + ListRuns(context.Context, *ListRunsRequest) (*ListRunsResponse, error) + // Requests cancellation. Cancelling a finished run changes nothing. + CancelRun(context.Context, *CancelRunRequest) (*CancelRunResponse, error) + ListRunResults(context.Context, *ListRunResultsRequest) (*ListRunResultsResponse, error) + // The run and its results, then each change, until the run reaches a terminal status. + WatchRun(*WatchRunRequest, grpc.ServerStreamingServer[WatchRunResponse]) error + mustEmbedUnimplementedRunServiceServer() +} + +// UnimplementedRunServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRunServiceServer struct{} + +func (UnimplementedRunServiceServer) SubmitRun(context.Context, *SubmitRunRequest) (*SubmitRunResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SubmitRun not implemented") +} +func (UnimplementedRunServiceServer) GetRun(context.Context, *GetRunRequest) (*GetRunResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetRun not implemented") +} +func (UnimplementedRunServiceServer) ListRuns(context.Context, *ListRunsRequest) (*ListRunsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListRuns not implemented") +} +func (UnimplementedRunServiceServer) CancelRun(context.Context, *CancelRunRequest) (*CancelRunResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CancelRun not implemented") +} +func (UnimplementedRunServiceServer) ListRunResults(context.Context, *ListRunResultsRequest) (*ListRunResultsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListRunResults not implemented") +} +func (UnimplementedRunServiceServer) WatchRun(*WatchRunRequest, grpc.ServerStreamingServer[WatchRunResponse]) error { + return status.Error(codes.Unimplemented, "method WatchRun not implemented") +} +func (UnimplementedRunServiceServer) mustEmbedUnimplementedRunServiceServer() {} +func (UnimplementedRunServiceServer) testEmbeddedByValue() {} + +// UnsafeRunServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RunServiceServer will +// result in compilation errors. +type UnsafeRunServiceServer interface { + mustEmbedUnimplementedRunServiceServer() +} + +func RegisterRunServiceServer(s grpc.ServiceRegistrar, srv RunServiceServer) { + // If the following call panics, it indicates UnimplementedRunServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&RunService_ServiceDesc, srv) +} + +func _RunService_SubmitRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubmitRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RunServiceServer).SubmitRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RunService_SubmitRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RunServiceServer).SubmitRun(ctx, req.(*SubmitRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _RunService_GetRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RunServiceServer).GetRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RunService_GetRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RunServiceServer).GetRun(ctx, req.(*GetRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _RunService_ListRuns_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListRunsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RunServiceServer).ListRuns(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RunService_ListRuns_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RunServiceServer).ListRuns(ctx, req.(*ListRunsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _RunService_CancelRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CancelRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RunServiceServer).CancelRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RunService_CancelRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RunServiceServer).CancelRun(ctx, req.(*CancelRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _RunService_ListRunResults_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListRunResultsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RunServiceServer).ListRunResults(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RunService_ListRunResults_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RunServiceServer).ListRunResults(ctx, req.(*ListRunResultsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _RunService_WatchRun_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WatchRunRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(RunServiceServer).WatchRun(m, &grpc.GenericServerStream[WatchRunRequest, WatchRunResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RunService_WatchRunServer = grpc.ServerStreamingServer[WatchRunResponse] + +// RunService_ServiceDesc is the grpc.ServiceDesc for RunService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RunService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agentevals.api.v1alpha1.RunService", + HandlerType: (*RunServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SubmitRun", + Handler: _RunService_SubmitRun_Handler, + }, + { + MethodName: "GetRun", + Handler: _RunService_GetRun_Handler, + }, + { + MethodName: "ListRuns", + Handler: _RunService_ListRuns_Handler, + }, + { + MethodName: "CancelRun", + Handler: _RunService_CancelRun_Handler, + }, + { + MethodName: "ListRunResults", + Handler: _RunService_ListRunResults_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "WatchRun", + Handler: _RunService_WatchRun_Handler, + ServerStreams: true, + }, + }, + Metadata: "agentevals/api/v1alpha1/runs.proto", +} diff --git a/gen/go/agentevals/api/v1alpha1/sessions.pb.go b/gen/go/agentevals/api/v1alpha1/sessions.pb.go new file mode 100644 index 00000000..7a1bdc23 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/sessions.pb.go @@ -0,0 +1,552 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/sessions.proto + +package apiv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ListSessionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Page *PageRequest `protobuf:"bytes,1,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListSessionsRequest) Reset() { + *x = ListSessionsRequest{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListSessionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListSessionsRequest) ProtoMessage() {} + +func (x *ListSessionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListSessionsRequest.ProtoReflect.Descriptor instead. +func (*ListSessionsRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{0} +} + +func (x *ListSessionsRequest) GetPage() *PageRequest { + if x != nil { + return x.Page + } + return nil +} + +type ListSessionsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Sessions []*SessionSummary `protobuf:"bytes,1,rep,name=sessions,proto3" json:"sessions,omitempty"` + Page *PageResponse `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListSessionsResponse) Reset() { + *x = ListSessionsResponse{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListSessionsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListSessionsResponse) ProtoMessage() {} + +func (x *ListSessionsResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListSessionsResponse.ProtoReflect.Descriptor instead. +func (*ListSessionsResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{1} +} + +func (x *ListSessionsResponse) GetSessions() []*SessionSummary { + if x != nil { + return x.Sessions + } + return nil +} + +func (x *ListSessionsResponse) GetPage() *PageResponse { + if x != nil { + return x.Page + } + return nil +} + +type GetSessionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSessionRequest) Reset() { + *x = GetSessionRequest{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSessionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSessionRequest) ProtoMessage() {} + +func (x *GetSessionRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSessionRequest.ProtoReflect.Descriptor instead. +func (*GetSessionRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{2} +} + +func (x *GetSessionRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type GetSessionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Session *Session `protobuf:"bytes,1,opt,name=session,proto3" json:"session,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSessionResponse) Reset() { + *x = GetSessionResponse{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSessionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSessionResponse) ProtoMessage() {} + +func (x *GetSessionResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSessionResponse.ProtoReflect.Descriptor instead. +func (*GetSessionResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{3} +} + +func (x *GetSessionResponse) GetSession() *Session { + if x != nil { + return x.Session + } + return nil +} + +type WatchSessionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Send each session with its turns. Otherwise upserts carry the summary only. + IncludeTurns bool `protobuf:"varint,1,opt,name=include_turns,json=includeTurns,proto3" json:"include_turns,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchSessionsRequest) Reset() { + *x = WatchSessionsRequest{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchSessionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchSessionsRequest) ProtoMessage() {} + +func (x *WatchSessionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchSessionsRequest.ProtoReflect.Descriptor instead. +func (*WatchSessionsRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{4} +} + +func (x *WatchSessionsRequest) GetIncludeTurns() bool { + if x != nil { + return x.IncludeTurns + } + return false +} + +type SessionRemoved struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The session that took over this session's traces, when the two were merged. + AbsorbedBy string `protobuf:"bytes,2,opt,name=absorbed_by,json=absorbedBy,proto3" json:"absorbed_by,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionRemoved) Reset() { + *x = SessionRemoved{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionRemoved) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionRemoved) ProtoMessage() {} + +func (x *SessionRemoved) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionRemoved.ProtoReflect.Descriptor instead. +func (*SessionRemoved) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{5} +} + +func (x *SessionRemoved) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SessionRemoved) GetAbsorbedBy() string { + if x != nil { + return x.AbsorbedBy + } + return "" +} + +// The end of the initial snapshot. +type Synced struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Synced) Reset() { + *x = Synced{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Synced) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Synced) ProtoMessage() {} + +func (x *Synced) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Synced.ProtoReflect.Descriptor instead. +func (*Synced) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{6} +} + +type WatchSessionsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Event: + // + // *WatchSessionsResponse_Upsert + // *WatchSessionsResponse_Removed + // *WatchSessionsResponse_Synced + Event isWatchSessionsResponse_Event `protobuf_oneof:"event"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchSessionsResponse) Reset() { + *x = WatchSessionsResponse{} + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchSessionsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchSessionsResponse) ProtoMessage() {} + +func (x *WatchSessionsResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_sessions_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchSessionsResponse.ProtoReflect.Descriptor instead. +func (*WatchSessionsResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP(), []int{7} +} + +func (x *WatchSessionsResponse) GetEvent() isWatchSessionsResponse_Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *WatchSessionsResponse) GetUpsert() *Session { + if x != nil { + if x, ok := x.Event.(*WatchSessionsResponse_Upsert); ok { + return x.Upsert + } + } + return nil +} + +func (x *WatchSessionsResponse) GetRemoved() *SessionRemoved { + if x != nil { + if x, ok := x.Event.(*WatchSessionsResponse_Removed); ok { + return x.Removed + } + } + return nil +} + +func (x *WatchSessionsResponse) GetSynced() *Synced { + if x != nil { + if x, ok := x.Event.(*WatchSessionsResponse_Synced); ok { + return x.Synced + } + } + return nil +} + +type isWatchSessionsResponse_Event interface { + isWatchSessionsResponse_Event() +} + +type WatchSessionsResponse_Upsert struct { + // The session's current state. It replaces every earlier upsert with the same id. Upserts + // can be coalesced, so a client sees the latest state, not every intermediate one. + Upsert *Session `protobuf:"bytes,1,opt,name=upsert,proto3,oneof"` +} + +type WatchSessionsResponse_Removed struct { + Removed *SessionRemoved `protobuf:"bytes,2,opt,name=removed,proto3,oneof"` +} + +type WatchSessionsResponse_Synced struct { + Synced *Synced `protobuf:"bytes,3,opt,name=synced,proto3,oneof"` +} + +func (*WatchSessionsResponse_Upsert) isWatchSessionsResponse_Event() {} + +func (*WatchSessionsResponse_Removed) isWatchSessionsResponse_Event() {} + +func (*WatchSessionsResponse_Synced) isWatchSessionsResponse_Event() {} + +var File_agentevals_api_v1alpha1_sessions_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_sessions_proto_rawDesc = "" + + "\n" + + "&agentevals/api/v1alpha1/sessions.proto\x12\x17agentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a#agentevals/api/v1alpha1/views.proto\"O\n" + + "\x13ListSessionsRequest\x128\n" + + "\x04page\x18\x01 \x01(\v2$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x96\x01\n" + + "\x14ListSessionsResponse\x12C\n" + + "\bsessions\x18\x01 \x03(\v2'.agentevals.api.v1alpha1.SessionSummaryR\bsessions\x129\n" + + "\x04page\x18\x02 \x01(\v2%.agentevals.api.v1alpha1.PageResponseR\x04page\"#\n" + + "\x11GetSessionRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"P\n" + + "\x12GetSessionResponse\x12:\n" + + "\asession\x18\x01 \x01(\v2 .agentevals.api.v1alpha1.SessionR\asession\";\n" + + "\x14WatchSessionsRequest\x12#\n" + + "\rinclude_turns\x18\x01 \x01(\bR\fincludeTurns\"A\n" + + "\x0eSessionRemoved\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1f\n" + + "\vabsorbed_by\x18\x02 \x01(\tR\n" + + "absorbedBy\"\b\n" + + "\x06Synced\"\xdc\x01\n" + + "\x15WatchSessionsResponse\x12:\n" + + "\x06upsert\x18\x01 \x01(\v2 .agentevals.api.v1alpha1.SessionH\x00R\x06upsert\x12C\n" + + "\aremoved\x18\x02 \x01(\v2'.agentevals.api.v1alpha1.SessionRemovedH\x00R\aremoved\x129\n" + + "\x06synced\x18\x03 \x01(\v2\x1f.agentevals.api.v1alpha1.SyncedH\x00R\x06syncedB\a\n" + + "\x05event2\xd6\x02\n" + + "\x0eSessionService\x12k\n" + + "\fListSessions\x12,.agentevals.api.v1alpha1.ListSessionsRequest\x1a-.agentevals.api.v1alpha1.ListSessionsResponse\x12e\n" + + "\n" + + "GetSession\x12*.agentevals.api.v1alpha1.GetSessionRequest\x1a+.agentevals.api.v1alpha1.GetSessionResponse\x12p\n" + + "\rWatchSessions\x12-.agentevals.api.v1alpha1.WatchSessionsRequest\x1a..agentevals.api.v1alpha1.WatchSessionsResponse0\x01BQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_sessions_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_sessions_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_sessions_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_sessions_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_sessions_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_sessions_proto_rawDesc), len(file_agentevals_api_v1alpha1_sessions_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_sessions_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_sessions_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_agentevals_api_v1alpha1_sessions_proto_goTypes = []any{ + (*ListSessionsRequest)(nil), // 0: agentevals.api.v1alpha1.ListSessionsRequest + (*ListSessionsResponse)(nil), // 1: agentevals.api.v1alpha1.ListSessionsResponse + (*GetSessionRequest)(nil), // 2: agentevals.api.v1alpha1.GetSessionRequest + (*GetSessionResponse)(nil), // 3: agentevals.api.v1alpha1.GetSessionResponse + (*WatchSessionsRequest)(nil), // 4: agentevals.api.v1alpha1.WatchSessionsRequest + (*SessionRemoved)(nil), // 5: agentevals.api.v1alpha1.SessionRemoved + (*Synced)(nil), // 6: agentevals.api.v1alpha1.Synced + (*WatchSessionsResponse)(nil), // 7: agentevals.api.v1alpha1.WatchSessionsResponse + (*PageRequest)(nil), // 8: agentevals.api.v1alpha1.PageRequest + (*SessionSummary)(nil), // 9: agentevals.api.v1alpha1.SessionSummary + (*PageResponse)(nil), // 10: agentevals.api.v1alpha1.PageResponse + (*Session)(nil), // 11: agentevals.api.v1alpha1.Session +} +var file_agentevals_api_v1alpha1_sessions_proto_depIdxs = []int32{ + 8, // 0: agentevals.api.v1alpha1.ListSessionsRequest.page:type_name -> agentevals.api.v1alpha1.PageRequest + 9, // 1: agentevals.api.v1alpha1.ListSessionsResponse.sessions:type_name -> agentevals.api.v1alpha1.SessionSummary + 10, // 2: agentevals.api.v1alpha1.ListSessionsResponse.page:type_name -> agentevals.api.v1alpha1.PageResponse + 11, // 3: agentevals.api.v1alpha1.GetSessionResponse.session:type_name -> agentevals.api.v1alpha1.Session + 11, // 4: agentevals.api.v1alpha1.WatchSessionsResponse.upsert:type_name -> agentevals.api.v1alpha1.Session + 5, // 5: agentevals.api.v1alpha1.WatchSessionsResponse.removed:type_name -> agentevals.api.v1alpha1.SessionRemoved + 6, // 6: agentevals.api.v1alpha1.WatchSessionsResponse.synced:type_name -> agentevals.api.v1alpha1.Synced + 0, // 7: agentevals.api.v1alpha1.SessionService.ListSessions:input_type -> agentevals.api.v1alpha1.ListSessionsRequest + 2, // 8: agentevals.api.v1alpha1.SessionService.GetSession:input_type -> agentevals.api.v1alpha1.GetSessionRequest + 4, // 9: agentevals.api.v1alpha1.SessionService.WatchSessions:input_type -> agentevals.api.v1alpha1.WatchSessionsRequest + 1, // 10: agentevals.api.v1alpha1.SessionService.ListSessions:output_type -> agentevals.api.v1alpha1.ListSessionsResponse + 3, // 11: agentevals.api.v1alpha1.SessionService.GetSession:output_type -> agentevals.api.v1alpha1.GetSessionResponse + 7, // 12: agentevals.api.v1alpha1.SessionService.WatchSessions:output_type -> agentevals.api.v1alpha1.WatchSessionsResponse + 10, // [10:13] is the sub-list for method output_type + 7, // [7:10] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_sessions_proto_init() } +func file_agentevals_api_v1alpha1_sessions_proto_init() { + if File_agentevals_api_v1alpha1_sessions_proto != nil { + return + } + file_agentevals_api_v1alpha1_common_proto_init() + file_agentevals_api_v1alpha1_views_proto_init() + file_agentevals_api_v1alpha1_sessions_proto_msgTypes[7].OneofWrappers = []any{ + (*WatchSessionsResponse_Upsert)(nil), + (*WatchSessionsResponse_Removed)(nil), + (*WatchSessionsResponse_Synced)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_sessions_proto_rawDesc), len(file_agentevals_api_v1alpha1_sessions_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agentevals_api_v1alpha1_sessions_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_sessions_proto_depIdxs, + MessageInfos: file_agentevals_api_v1alpha1_sessions_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_sessions_proto = out.File + file_agentevals_api_v1alpha1_sessions_proto_goTypes = nil + file_agentevals_api_v1alpha1_sessions_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/sessions_grpc.pb.go b/gen/go/agentevals/api/v1alpha1/sessions_grpc.pb.go new file mode 100644 index 00000000..088e1e1c --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/sessions_grpc.pb.go @@ -0,0 +1,211 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: agentevals/api/v1alpha1/sessions.proto + +package apiv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + SessionService_ListSessions_FullMethodName = "/agentevals.api.v1alpha1.SessionService/ListSessions" + SessionService_GetSession_FullMethodName = "/agentevals.api.v1alpha1.SessionService/GetSession" + SessionService_WatchSessions_FullMethodName = "/agentevals.api.v1alpha1.SessionService/WatchSessions" +) + +// SessionServiceClient is the client API for SessionService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in +// memory and expire. +type SessionServiceClient interface { + // Sessions newest first, summaries only. + ListSessions(ctx context.Context, in *ListSessionsRequest, opts ...grpc.CallOption) (*ListSessionsResponse, error) + GetSession(ctx context.Context, in *GetSessionRequest, opts ...grpc.CallOption) (*GetSessionResponse, error) + // Every current session, then synced, then each change as it happens. + WatchSessions(ctx context.Context, in *WatchSessionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchSessionsResponse], error) +} + +type sessionServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewSessionServiceClient(cc grpc.ClientConnInterface) SessionServiceClient { + return &sessionServiceClient{cc} +} + +func (c *sessionServiceClient) ListSessions(ctx context.Context, in *ListSessionsRequest, opts ...grpc.CallOption) (*ListSessionsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListSessionsResponse) + err := c.cc.Invoke(ctx, SessionService_ListSessions_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sessionServiceClient) GetSession(ctx context.Context, in *GetSessionRequest, opts ...grpc.CallOption) (*GetSessionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSessionResponse) + err := c.cc.Invoke(ctx, SessionService_GetSession_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sessionServiceClient) WatchSessions(ctx context.Context, in *WatchSessionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchSessionsResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SessionService_ServiceDesc.Streams[0], SessionService_WatchSessions_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[WatchSessionsRequest, WatchSessionsResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SessionService_WatchSessionsClient = grpc.ServerStreamingClient[WatchSessionsResponse] + +// SessionServiceServer is the server API for SessionService service. +// All implementations must embed UnimplementedSessionServiceServer +// for forward compatibility. +// +// Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in +// memory and expire. +type SessionServiceServer interface { + // Sessions newest first, summaries only. + ListSessions(context.Context, *ListSessionsRequest) (*ListSessionsResponse, error) + GetSession(context.Context, *GetSessionRequest) (*GetSessionResponse, error) + // Every current session, then synced, then each change as it happens. + WatchSessions(*WatchSessionsRequest, grpc.ServerStreamingServer[WatchSessionsResponse]) error + mustEmbedUnimplementedSessionServiceServer() +} + +// UnimplementedSessionServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSessionServiceServer struct{} + +func (UnimplementedSessionServiceServer) ListSessions(context.Context, *ListSessionsRequest) (*ListSessionsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListSessions not implemented") +} +func (UnimplementedSessionServiceServer) GetSession(context.Context, *GetSessionRequest) (*GetSessionResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSession not implemented") +} +func (UnimplementedSessionServiceServer) WatchSessions(*WatchSessionsRequest, grpc.ServerStreamingServer[WatchSessionsResponse]) error { + return status.Error(codes.Unimplemented, "method WatchSessions not implemented") +} +func (UnimplementedSessionServiceServer) mustEmbedUnimplementedSessionServiceServer() {} +func (UnimplementedSessionServiceServer) testEmbeddedByValue() {} + +// UnsafeSessionServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SessionServiceServer will +// result in compilation errors. +type UnsafeSessionServiceServer interface { + mustEmbedUnimplementedSessionServiceServer() +} + +func RegisterSessionServiceServer(s grpc.ServiceRegistrar, srv SessionServiceServer) { + // If the following call panics, it indicates UnimplementedSessionServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&SessionService_ServiceDesc, srv) +} + +func _SessionService_ListSessions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListSessionsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SessionServiceServer).ListSessions(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SessionService_ListSessions_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SessionServiceServer).ListSessions(ctx, req.(*ListSessionsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SessionService_GetSession_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSessionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SessionServiceServer).GetSession(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SessionService_GetSession_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SessionServiceServer).GetSession(ctx, req.(*GetSessionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SessionService_WatchSessions_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WatchSessionsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SessionServiceServer).WatchSessions(m, &grpc.GenericServerStream[WatchSessionsRequest, WatchSessionsResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SessionService_WatchSessionsServer = grpc.ServerStreamingServer[WatchSessionsResponse] + +// SessionService_ServiceDesc is the grpc.ServiceDesc for SessionService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SessionService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agentevals.api.v1alpha1.SessionService", + HandlerType: (*SessionServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ListSessions", + Handler: _SessionService_ListSessions_Handler, + }, + { + MethodName: "GetSession", + Handler: _SessionService_GetSession_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "WatchSessions", + Handler: _SessionService_WatchSessions_Handler, + ServerStreams: true, + }, + }, + Metadata: "agentevals/api/v1alpha1/sessions.proto", +} diff --git a/gen/go/agentevals/api/v1alpha1/system.pb.go b/gen/go/agentevals/api/v1alpha1/system.pb.go new file mode 100644 index 00000000..4ee37a3a --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/system.pb.go @@ -0,0 +1,555 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/system.proto + +package apiv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GetVersionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetVersionRequest) Reset() { + *x = GetVersionRequest{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetVersionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetVersionRequest) ProtoMessage() {} + +func (x *GetVersionRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetVersionRequest.ProtoReflect.Descriptor instead. +func (*GetVersionRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{0} +} + +type GetVersionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetVersionResponse) Reset() { + *x = GetVersionResponse{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetVersionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetVersionResponse) ProtoMessage() {} + +func (x *GetVersionResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetVersionResponse.ProtoReflect.Descriptor instead. +func (*GetVersionResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{1} +} + +func (x *GetVersionResponse) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type GetConfigRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetConfigRequest) Reset() { + *x = GetConfigRequest{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetConfigRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetConfigRequest) ProtoMessage() {} + +func (x *GetConfigRequest) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetConfigRequest.ProtoReflect.Descriptor instead. +func (*GetConfigRequest) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{2} +} + +// Listener ports. 0 when a listener is off. +type Ports struct { + state protoimpl.MessageState `protogen:"open.v1"` + Http int32 `protobuf:"varint,1,opt,name=http,proto3" json:"http,omitempty"` + OtlpHttp int32 `protobuf:"varint,2,opt,name=otlp_http,json=otlpHttp,proto3" json:"otlp_http,omitempty"` + OtlpGrpc int32 `protobuf:"varint,3,opt,name=otlp_grpc,json=otlpGrpc,proto3" json:"otlp_grpc,omitempty"` + ApiGrpc int32 `protobuf:"varint,4,opt,name=api_grpc,json=apiGrpc,proto3" json:"api_grpc,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ports) Reset() { + *x = Ports{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ports) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ports) ProtoMessage() {} + +func (x *Ports) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ports.ProtoReflect.Descriptor instead. +func (*Ports) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{3} +} + +func (x *Ports) GetHttp() int32 { + if x != nil { + return x.Http + } + return 0 +} + +func (x *Ports) GetOtlpHttp() int32 { + if x != nil { + return x.OtlpHttp + } + return 0 +} + +func (x *Ports) GetOtlpGrpc() int32 { + if x != nil { + return x.OtlpGrpc + } + return 0 +} + +func (x *Ports) GetApiGrpc() int32 { + if x != nil { + return x.ApiGrpc + } + return 0 +} + +// Bounds of the live session store. Telemetry over a bound is rejected or evicted. +type LiveLimits struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxSessions int32 `protobuf:"varint,1,opt,name=max_sessions,json=maxSessions,proto3" json:"max_sessions,omitempty"` + SpansPerTrace int32 `protobuf:"varint,2,opt,name=spans_per_trace,json=spansPerTrace,proto3" json:"spans_per_trace,omitempty"` + SpansPerSession int32 `protobuf:"varint,3,opt,name=spans_per_session,json=spansPerSession,proto3" json:"spans_per_session,omitempty"` + TracesPerSession int32 `protobuf:"varint,4,opt,name=traces_per_session,json=tracesPerSession,proto3" json:"traces_per_session,omitempty"` + MaxBytes int64 `protobuf:"varint,5,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + CompletionGrace *durationpb.Duration `protobuf:"bytes,6,opt,name=completion_grace,json=completionGrace,proto3" json:"completion_grace,omitempty"` + IdleTimeout *durationpb.Duration `protobuf:"bytes,7,opt,name=idle_timeout,json=idleTimeout,proto3" json:"idle_timeout,omitempty"` + SessionTtl *durationpb.Duration `protobuf:"bytes,8,opt,name=session_ttl,json=sessionTtl,proto3" json:"session_ttl,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LiveLimits) Reset() { + *x = LiveLimits{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LiveLimits) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LiveLimits) ProtoMessage() {} + +func (x *LiveLimits) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LiveLimits.ProtoReflect.Descriptor instead. +func (*LiveLimits) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{4} +} + +func (x *LiveLimits) GetMaxSessions() int32 { + if x != nil { + return x.MaxSessions + } + return 0 +} + +func (x *LiveLimits) GetSpansPerTrace() int32 { + if x != nil { + return x.SpansPerTrace + } + return 0 +} + +func (x *LiveLimits) GetSpansPerSession() int32 { + if x != nil { + return x.SpansPerSession + } + return 0 +} + +func (x *LiveLimits) GetTracesPerSession() int32 { + if x != nil { + return x.TracesPerSession + } + return 0 +} + +func (x *LiveLimits) GetMaxBytes() int64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +func (x *LiveLimits) GetCompletionGrace() *durationpb.Duration { + if x != nil { + return x.CompletionGrace + } + return nil +} + +func (x *LiveLimits) GetIdleTimeout() *durationpb.Duration { + if x != nil { + return x.IdleTimeout + } + return nil +} + +func (x *LiveLimits) GetSessionTtl() *durationpb.Duration { + if x != nil { + return x.SessionTtl + } + return nil +} + +// Which judge provider API keys the server has in its environment. Values are never returned. +type JudgeKeys struct { + state protoimpl.MessageState `protogen:"open.v1"` + Google bool `protobuf:"varint,1,opt,name=google,proto3" json:"google,omitempty"` + Anthropic bool `protobuf:"varint,2,opt,name=anthropic,proto3" json:"anthropic,omitempty"` + Openai bool `protobuf:"varint,3,opt,name=openai,proto3" json:"openai,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *JudgeKeys) Reset() { + *x = JudgeKeys{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *JudgeKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*JudgeKeys) ProtoMessage() {} + +func (x *JudgeKeys) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use JudgeKeys.ProtoReflect.Descriptor instead. +func (*JudgeKeys) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{5} +} + +func (x *JudgeKeys) GetGoogle() bool { + if x != nil { + return x.Google + } + return false +} + +func (x *JudgeKeys) GetAnthropic() bool { + if x != nil { + return x.Anthropic + } + return false +} + +func (x *JudgeKeys) GetOpenai() bool { + if x != nil { + return x.Openai + } + return false +} + +type GetConfigResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ports *Ports `protobuf:"bytes,1,opt,name=ports,proto3" json:"ports,omitempty"` + LiveLimits *LiveLimits `protobuf:"bytes,2,opt,name=live_limits,json=liveLimits,proto3" json:"live_limits,omitempty"` + // Whether run storage is configured, so RunService works. + RunsEnabled bool `protobuf:"varint,3,opt,name=runs_enabled,json=runsEnabled,proto3" json:"runs_enabled,omitempty"` + // Whether evaluation results are emitted as gen_ai.evaluation.result events. + EvaluationEventsEnabled bool `protobuf:"varint,4,opt,name=evaluation_events_enabled,json=evaluationEventsEnabled,proto3" json:"evaluation_events_enabled,omitempty"` + JudgeKeys *JudgeKeys `protobuf:"bytes,5,opt,name=judge_keys,json=judgeKeys,proto3" json:"judge_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetConfigResponse) Reset() { + *x = GetConfigResponse{} + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetConfigResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetConfigResponse) ProtoMessage() {} + +func (x *GetConfigResponse) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_system_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetConfigResponse.ProtoReflect.Descriptor instead. +func (*GetConfigResponse) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_system_proto_rawDescGZIP(), []int{6} +} + +func (x *GetConfigResponse) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +func (x *GetConfigResponse) GetLiveLimits() *LiveLimits { + if x != nil { + return x.LiveLimits + } + return nil +} + +func (x *GetConfigResponse) GetRunsEnabled() bool { + if x != nil { + return x.RunsEnabled + } + return false +} + +func (x *GetConfigResponse) GetEvaluationEventsEnabled() bool { + if x != nil { + return x.EvaluationEventsEnabled + } + return false +} + +func (x *GetConfigResponse) GetJudgeKeys() *JudgeKeys { + if x != nil { + return x.JudgeKeys + } + return nil +} + +var File_agentevals_api_v1alpha1_system_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_system_proto_rawDesc = "" + + "\n" + + "$agentevals/api/v1alpha1/system.proto\x12\x17agentevals.api.v1alpha1\x1a\x1egoogle/protobuf/duration.proto\"\x13\n" + + "\x11GetVersionRequest\".\n" + + "\x12GetVersionResponse\x12\x18\n" + + "\aversion\x18\x01 \x01(\tR\aversion\"\x12\n" + + "\x10GetConfigRequest\"p\n" + + "\x05Ports\x12\x12\n" + + "\x04http\x18\x01 \x01(\x05R\x04http\x12\x1b\n" + + "\totlp_http\x18\x02 \x01(\x05R\botlpHttp\x12\x1b\n" + + "\totlp_grpc\x18\x03 \x01(\x05R\botlpGrpc\x12\x19\n" + + "\bapi_grpc\x18\x04 \x01(\x05R\aapiGrpc\"\x8e\x03\n" + + "\n" + + "LiveLimits\x12!\n" + + "\fmax_sessions\x18\x01 \x01(\x05R\vmaxSessions\x12&\n" + + "\x0fspans_per_trace\x18\x02 \x01(\x05R\rspansPerTrace\x12*\n" + + "\x11spans_per_session\x18\x03 \x01(\x05R\x0fspansPerSession\x12,\n" + + "\x12traces_per_session\x18\x04 \x01(\x05R\x10tracesPerSession\x12\x1b\n" + + "\tmax_bytes\x18\x05 \x01(\x03R\bmaxBytes\x12D\n" + + "\x10completion_grace\x18\x06 \x01(\v2\x19.google.protobuf.DurationR\x0fcompletionGrace\x12<\n" + + "\fidle_timeout\x18\a \x01(\v2\x19.google.protobuf.DurationR\vidleTimeout\x12:\n" + + "\vsession_ttl\x18\b \x01(\v2\x19.google.protobuf.DurationR\n" + + "sessionTtl\"Y\n" + + "\tJudgeKeys\x12\x16\n" + + "\x06google\x18\x01 \x01(\bR\x06google\x12\x1c\n" + + "\tanthropic\x18\x02 \x01(\bR\tanthropic\x12\x16\n" + + "\x06openai\x18\x03 \x01(\bR\x06openai\"\xb1\x02\n" + + "\x11GetConfigResponse\x124\n" + + "\x05ports\x18\x01 \x01(\v2\x1e.agentevals.api.v1alpha1.PortsR\x05ports\x12D\n" + + "\vlive_limits\x18\x02 \x01(\v2#.agentevals.api.v1alpha1.LiveLimitsR\n" + + "liveLimits\x12!\n" + + "\fruns_enabled\x18\x03 \x01(\bR\vrunsEnabled\x12:\n" + + "\x19evaluation_events_enabled\x18\x04 \x01(\bR\x17evaluationEventsEnabled\x12A\n" + + "\n" + + "judge_keys\x18\x05 \x01(\v2\".agentevals.api.v1alpha1.JudgeKeysR\tjudgeKeys2\xda\x01\n" + + "\rSystemService\x12e\n" + + "\n" + + "GetVersion\x12*.agentevals.api.v1alpha1.GetVersionRequest\x1a+.agentevals.api.v1alpha1.GetVersionResponse\x12b\n" + + "\tGetConfig\x12).agentevals.api.v1alpha1.GetConfigRequest\x1a*.agentevals.api.v1alpha1.GetConfigResponseBQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_system_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_system_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_system_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_system_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_system_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_system_proto_rawDesc), len(file_agentevals_api_v1alpha1_system_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_system_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_system_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_agentevals_api_v1alpha1_system_proto_goTypes = []any{ + (*GetVersionRequest)(nil), // 0: agentevals.api.v1alpha1.GetVersionRequest + (*GetVersionResponse)(nil), // 1: agentevals.api.v1alpha1.GetVersionResponse + (*GetConfigRequest)(nil), // 2: agentevals.api.v1alpha1.GetConfigRequest + (*Ports)(nil), // 3: agentevals.api.v1alpha1.Ports + (*LiveLimits)(nil), // 4: agentevals.api.v1alpha1.LiveLimits + (*JudgeKeys)(nil), // 5: agentevals.api.v1alpha1.JudgeKeys + (*GetConfigResponse)(nil), // 6: agentevals.api.v1alpha1.GetConfigResponse + (*durationpb.Duration)(nil), // 7: google.protobuf.Duration +} +var file_agentevals_api_v1alpha1_system_proto_depIdxs = []int32{ + 7, // 0: agentevals.api.v1alpha1.LiveLimits.completion_grace:type_name -> google.protobuf.Duration + 7, // 1: agentevals.api.v1alpha1.LiveLimits.idle_timeout:type_name -> google.protobuf.Duration + 7, // 2: agentevals.api.v1alpha1.LiveLimits.session_ttl:type_name -> google.protobuf.Duration + 3, // 3: agentevals.api.v1alpha1.GetConfigResponse.ports:type_name -> agentevals.api.v1alpha1.Ports + 4, // 4: agentevals.api.v1alpha1.GetConfigResponse.live_limits:type_name -> agentevals.api.v1alpha1.LiveLimits + 5, // 5: agentevals.api.v1alpha1.GetConfigResponse.judge_keys:type_name -> agentevals.api.v1alpha1.JudgeKeys + 0, // 6: agentevals.api.v1alpha1.SystemService.GetVersion:input_type -> agentevals.api.v1alpha1.GetVersionRequest + 2, // 7: agentevals.api.v1alpha1.SystemService.GetConfig:input_type -> agentevals.api.v1alpha1.GetConfigRequest + 1, // 8: agentevals.api.v1alpha1.SystemService.GetVersion:output_type -> agentevals.api.v1alpha1.GetVersionResponse + 6, // 9: agentevals.api.v1alpha1.SystemService.GetConfig:output_type -> agentevals.api.v1alpha1.GetConfigResponse + 8, // [8:10] is the sub-list for method output_type + 6, // [6:8] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_system_proto_init() } +func file_agentevals_api_v1alpha1_system_proto_init() { + if File_agentevals_api_v1alpha1_system_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_system_proto_rawDesc), len(file_agentevals_api_v1alpha1_system_proto_rawDesc)), + NumEnums: 0, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agentevals_api_v1alpha1_system_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_system_proto_depIdxs, + MessageInfos: file_agentevals_api_v1alpha1_system_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_system_proto = out.File + file_agentevals_api_v1alpha1_system_proto_goTypes = nil + file_agentevals_api_v1alpha1_system_proto_depIdxs = nil +} diff --git a/gen/go/agentevals/api/v1alpha1/system_grpc.pb.go b/gen/go/agentevals/api/v1alpha1/system_grpc.pb.go new file mode 100644 index 00000000..a16bfec4 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/system_grpc.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: agentevals/api/v1alpha1/system.proto + +package apiv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + SystemService_GetVersion_FullMethodName = "/agentevals.api.v1alpha1.SystemService/GetVersion" + SystemService_GetConfig_FullMethodName = "/agentevals.api.v1alpha1.SystemService/GetConfig" +) + +// SystemServiceClient is the client API for SystemService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SystemServiceClient interface { + GetVersion(ctx context.Context, in *GetVersionRequest, opts ...grpc.CallOption) (*GetVersionResponse, error) + GetConfig(ctx context.Context, in *GetConfigRequest, opts ...grpc.CallOption) (*GetConfigResponse, error) +} + +type systemServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewSystemServiceClient(cc grpc.ClientConnInterface) SystemServiceClient { + return &systemServiceClient{cc} +} + +func (c *systemServiceClient) GetVersion(ctx context.Context, in *GetVersionRequest, opts ...grpc.CallOption) (*GetVersionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetVersionResponse) + err := c.cc.Invoke(ctx, SystemService_GetVersion_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *systemServiceClient) GetConfig(ctx context.Context, in *GetConfigRequest, opts ...grpc.CallOption) (*GetConfigResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetConfigResponse) + err := c.cc.Invoke(ctx, SystemService_GetConfig_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SystemServiceServer is the server API for SystemService service. +// All implementations must embed UnimplementedSystemServiceServer +// for forward compatibility. +type SystemServiceServer interface { + GetVersion(context.Context, *GetVersionRequest) (*GetVersionResponse, error) + GetConfig(context.Context, *GetConfigRequest) (*GetConfigResponse, error) + mustEmbedUnimplementedSystemServiceServer() +} + +// UnimplementedSystemServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSystemServiceServer struct{} + +func (UnimplementedSystemServiceServer) GetVersion(context.Context, *GetVersionRequest) (*GetVersionResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetVersion not implemented") +} +func (UnimplementedSystemServiceServer) GetConfig(context.Context, *GetConfigRequest) (*GetConfigResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetConfig not implemented") +} +func (UnimplementedSystemServiceServer) mustEmbedUnimplementedSystemServiceServer() {} +func (UnimplementedSystemServiceServer) testEmbeddedByValue() {} + +// UnsafeSystemServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SystemServiceServer will +// result in compilation errors. +type UnsafeSystemServiceServer interface { + mustEmbedUnimplementedSystemServiceServer() +} + +func RegisterSystemServiceServer(s grpc.ServiceRegistrar, srv SystemServiceServer) { + // If the following call panics, it indicates UnimplementedSystemServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&SystemService_ServiceDesc, srv) +} + +func _SystemService_GetVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVersionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SystemServiceServer).GetVersion(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SystemService_GetVersion_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SystemServiceServer).GetVersion(ctx, req.(*GetVersionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SystemService_GetConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetConfigRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SystemServiceServer).GetConfig(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SystemService_GetConfig_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SystemServiceServer).GetConfig(ctx, req.(*GetConfigRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SystemService_ServiceDesc is the grpc.ServiceDesc for SystemService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SystemService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agentevals.api.v1alpha1.SystemService", + HandlerType: (*SystemServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetVersion", + Handler: _SystemService_GetVersion_Handler, + }, + { + MethodName: "GetConfig", + Handler: _SystemService_GetConfig_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "agentevals/api/v1alpha1/system.proto", +} diff --git a/gen/go/agentevals/api/v1alpha1/views.pb.go b/gen/go/agentevals/api/v1alpha1/views.pb.go new file mode 100644 index 00000000..59f93e01 --- /dev/null +++ b/gen/go/agentevals/api/v1alpha1/views.pb.go @@ -0,0 +1,1491 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agentevals/api/v1alpha1/views.proto + +package apiv1alpha1 + +import ( + v11 "go.opentelemetry.io/proto/otlp/resource/v1" + v1 "go.opentelemetry.io/proto/otlp/trace/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + structpb "google.golang.org/protobuf/types/known/structpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SessionState int32 + +const ( + SessionState_SESSION_STATE_UNSPECIFIED SessionState = 0 + // Spans may still arrive. + SessionState_SESSION_STATE_LIVE SessionState = 1 + // The completion grace or the idle timeout has passed. New spans reopen the session. + SessionState_SESSION_STATE_COMPLETE SessionState = 2 +) + +// Enum value maps for SessionState. +var ( + SessionState_name = map[int32]string{ + 0: "SESSION_STATE_UNSPECIFIED", + 1: "SESSION_STATE_LIVE", + 2: "SESSION_STATE_COMPLETE", + } + SessionState_value = map[string]int32{ + "SESSION_STATE_UNSPECIFIED": 0, + "SESSION_STATE_LIVE": 1, + "SESSION_STATE_COMPLETE": 2, + } +) + +func (x SessionState) Enum() *SessionState { + p := new(SessionState) + *p = x + return p +} + +func (x SessionState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SessionState) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_views_proto_enumTypes[0].Descriptor() +} + +func (SessionState) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_views_proto_enumTypes[0] +} + +func (x SessionState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SessionState.Descriptor instead. +func (SessionState) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{0} +} + +type ContentCapture int32 + +const ( + // No turns yet. + ContentCapture_CONTENT_CAPTURE_UNSPECIFIED ContentCapture = 0 + ContentCapture_CONTENT_CAPTURE_ON ContentCapture = 1 + ContentCapture_CONTENT_CAPTURE_OFF ContentCapture = 2 + // Some turns have content, others do not. + ContentCapture_CONTENT_CAPTURE_PARTIAL ContentCapture = 3 +) + +// Enum value maps for ContentCapture. +var ( + ContentCapture_name = map[int32]string{ + 0: "CONTENT_CAPTURE_UNSPECIFIED", + 1: "CONTENT_CAPTURE_ON", + 2: "CONTENT_CAPTURE_OFF", + 3: "CONTENT_CAPTURE_PARTIAL", + } + ContentCapture_value = map[string]int32{ + "CONTENT_CAPTURE_UNSPECIFIED": 0, + "CONTENT_CAPTURE_ON": 1, + "CONTENT_CAPTURE_OFF": 2, + "CONTENT_CAPTURE_PARTIAL": 3, + } +) + +func (x ContentCapture) Enum() *ContentCapture { + p := new(ContentCapture) + *p = x + return p +} + +func (x ContentCapture) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ContentCapture) Descriptor() protoreflect.EnumDescriptor { + return file_agentevals_api_v1alpha1_views_proto_enumTypes[1].Descriptor() +} + +func (ContentCapture) Type() protoreflect.EnumType { + return &file_agentevals_api_v1alpha1_views_proto_enumTypes[1] +} + +func (x ContentCapture) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ContentCapture.Descriptor instead. +func (ContentCapture) EnumDescriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{1} +} + +// Token counts. Unset means not recorded; 0 means zero. +type Usage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gen_ai.usage.input_tokens, which includes cached input tokens. + InputTokens *int64 `protobuf:"varint,1,opt,name=input_tokens,json=inputTokens,proto3,oneof" json:"input_tokens,omitempty"` + // gen_ai.usage.output_tokens. + OutputTokens *int64 `protobuf:"varint,2,opt,name=output_tokens,json=outputTokens,proto3,oneof" json:"output_tokens,omitempty"` + // gen_ai.usage.cache_read.input_tokens. + CacheReadInputTokens *int64 `protobuf:"varint,3,opt,name=cache_read_input_tokens,json=cacheReadInputTokens,proto3,oneof" json:"cache_read_input_tokens,omitempty"` + // gen_ai.usage.cache_write.input_tokens. + CacheWriteInputTokens *int64 `protobuf:"varint,4,opt,name=cache_write_input_tokens,json=cacheWriteInputTokens,proto3,oneof" json:"cache_write_input_tokens,omitempty"` + // gen_ai.usage.reasoning.output_tokens. + ReasoningOutputTokens *int64 `protobuf:"varint,5,opt,name=reasoning_output_tokens,json=reasoningOutputTokens,proto3,oneof" json:"reasoning_output_tokens,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Usage) Reset() { + *x = Usage{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Usage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Usage) ProtoMessage() {} + +func (x *Usage) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Usage.ProtoReflect.Descriptor instead. +func (*Usage) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{0} +} + +func (x *Usage) GetInputTokens() int64 { + if x != nil && x.InputTokens != nil { + return *x.InputTokens + } + return 0 +} + +func (x *Usage) GetOutputTokens() int64 { + if x != nil && x.OutputTokens != nil { + return *x.OutputTokens + } + return 0 +} + +func (x *Usage) GetCacheReadInputTokens() int64 { + if x != nil && x.CacheReadInputTokens != nil { + return *x.CacheReadInputTokens + } + return 0 +} + +func (x *Usage) GetCacheWriteInputTokens() int64 { + if x != nil && x.CacheWriteInputTokens != nil { + return *x.CacheWriteInputTokens + } + return 0 +} + +func (x *Usage) GetReasoningOutputTokens() int64 { + if x != nil && x.ReasoningOutputTokens != nil { + return *x.ReasoningOutputTokens + } + return 0 +} + +// Why something is missing or approximate. Codes are stable; reasons are for people and never +// carry message content. +type Gap struct { + state protoimpl.MessageState `protogen:"open.v1"` + Code string `protobuf:"bytes,1,opt,name=code,proto3" json:"code,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Gap) Reset() { + *x = Gap{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Gap) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Gap) ProtoMessage() {} + +func (x *Gap) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Gap.ProtoReflect.Descriptor instead. +func (*Gap) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{1} +} + +func (x *Gap) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *Gap) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +// An evaluation result with the fields of the gen_ai.evaluation.result event. +type EvaluationResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gen_ai.evaluation.name. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // gen_ai.evaluation.score.value. + ScoreValue *float64 `protobuf:"fixed64,2,opt,name=score_value,json=scoreValue,proto3,oneof" json:"score_value,omitempty"` + // gen_ai.evaluation.score.label. + ScoreLabel *string `protobuf:"bytes,3,opt,name=score_label,json=scoreLabel,proto3,oneof" json:"score_label,omitempty"` + // gen_ai.evaluation.explanation. + Explanation *string `protobuf:"bytes,4,opt,name=explanation,proto3,oneof" json:"explanation,omitempty"` + // error.type, when the evaluation itself failed. + ErrorType *string `protobuf:"bytes,5,opt,name=error_type,json=errorType,proto3,oneof" json:"error_type,omitempty"` + // The judged span. + TraceId string `protobuf:"bytes,6,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanId string `protobuf:"bytes,7,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + // gen_ai.response.id of the judged response. + ResponseId string `protobuf:"bytes,8,opt,name=response_id,json=responseId,proto3" json:"response_id,omitempty"` + // service.name of the producer, for results found in telemetry. + SourceService string `protobuf:"bytes,9,opt,name=source_service,json=sourceService,proto3" json:"source_service,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluationResult) Reset() { + *x = EvaluationResult{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluationResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluationResult) ProtoMessage() {} + +func (x *EvaluationResult) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluationResult.ProtoReflect.Descriptor instead. +func (*EvaluationResult) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{2} +} + +func (x *EvaluationResult) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EvaluationResult) GetScoreValue() float64 { + if x != nil && x.ScoreValue != nil { + return *x.ScoreValue + } + return 0 +} + +func (x *EvaluationResult) GetScoreLabel() string { + if x != nil && x.ScoreLabel != nil { + return *x.ScoreLabel + } + return "" +} + +func (x *EvaluationResult) GetExplanation() string { + if x != nil && x.Explanation != nil { + return *x.Explanation + } + return "" +} + +func (x *EvaluationResult) GetErrorType() string { + if x != nil && x.ErrorType != nil { + return *x.ErrorType + } + return "" +} + +func (x *EvaluationResult) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *EvaluationResult) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +func (x *EvaluationResult) GetResponseId() string { + if x != nil { + return x.ResponseId + } + return "" +} + +func (x *EvaluationResult) GetSourceService() string { + if x != nil { + return x.SourceService + } + return "" +} + +type ToolCallView struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The outermost span of the call. Empty when the call is only known from message parts. + SpanId string `protobuf:"bytes,1,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + // gen_ai.tool.name. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // gen_ai.tool.call.id. + CallId *string `protobuf:"bytes,3,opt,name=call_id,json=callId,proto3,oneof" json:"call_id,omitempty"` + // gen_ai.tool.call.arguments, parsed. Unset when not recorded. + Arguments *structpb.Value `protobuf:"bytes,4,opt,name=arguments,proto3" json:"arguments,omitempty"` + // gen_ai.tool.call.result, parsed, paired with the call by call id and then by order. Unset + // when not recorded. + Result *structpb.Value `protobuf:"bytes,5,opt,name=result,proto3" json:"result,omitempty"` + IsError bool `protobuf:"varint,6,opt,name=is_error,json=isError,proto3" json:"is_error,omitempty"` + // error.type. + ErrorType string `protobuf:"bytes,7,opt,name=error_type,json=errorType,proto3" json:"error_type,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + Duration *durationpb.Duration `protobuf:"bytes,9,opt,name=duration,proto3" json:"duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ToolCallView) Reset() { + *x = ToolCallView{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ToolCallView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ToolCallView) ProtoMessage() {} + +func (x *ToolCallView) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ToolCallView.ProtoReflect.Descriptor instead. +func (*ToolCallView) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{3} +} + +func (x *ToolCallView) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +func (x *ToolCallView) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ToolCallView) GetCallId() string { + if x != nil && x.CallId != nil { + return *x.CallId + } + return "" +} + +func (x *ToolCallView) GetArguments() *structpb.Value { + if x != nil { + return x.Arguments + } + return nil +} + +func (x *ToolCallView) GetResult() *structpb.Value { + if x != nil { + return x.Result + } + return nil +} + +func (x *ToolCallView) GetIsError() bool { + if x != nil { + return x.IsError + } + return false +} + +func (x *ToolCallView) GetErrorType() string { + if x != nil { + return x.ErrorType + } + return "" +} + +func (x *ToolCallView) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *ToolCallView) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +// One logical model call: a leaf inference span with its single leaf wrappers. +type ModelCallView struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpanId string `protobuf:"bytes,1,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + // gen_ai.operation.name. + Operation string `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` + // gen_ai.provider.name. + Provider string `protobuf:"bytes,3,opt,name=provider,proto3" json:"provider,omitempty"` + // gen_ai.request.model. + RequestModel string `protobuf:"bytes,4,opt,name=request_model,json=requestModel,proto3" json:"request_model,omitempty"` + // gen_ai.response.model. + ResponseModel string `protobuf:"bytes,5,opt,name=response_model,json=responseModel,proto3" json:"response_model,omitempty"` + // gen_ai.response.id. + ResponseId string `protobuf:"bytes,6,opt,name=response_id,json=responseId,proto3" json:"response_id,omitempty"` + Usage *Usage `protobuf:"bytes,7,opt,name=usage,proto3" json:"usage,omitempty"` + // gen_ai.response.finish_reasons. + FinishReasons []string `protobuf:"bytes,8,rep,name=finish_reasons,json=finishReasons,proto3" json:"finish_reasons,omitempty"` + // The span status. + Status v1.Status_StatusCode `protobuf:"varint,9,opt,name=status,proto3,enum=opentelemetry.proto.trace.v1.Status_StatusCode" json:"status,omitempty"` + // error.type. + ErrorType string `protobuf:"bytes,10,opt,name=error_type,json=errorType,proto3" json:"error_type,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + Duration *durationpb.Duration `protobuf:"bytes,12,opt,name=duration,proto3" json:"duration,omitempty"` + // gen_ai.agent.name of the agent that made the call. + AgentName string `protobuf:"bytes,13,opt,name=agent_name,json=agentName,proto3" json:"agent_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ModelCallView) Reset() { + *x = ModelCallView{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ModelCallView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelCallView) ProtoMessage() {} + +func (x *ModelCallView) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelCallView.ProtoReflect.Descriptor instead. +func (*ModelCallView) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{4} +} + +func (x *ModelCallView) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +func (x *ModelCallView) GetOperation() string { + if x != nil { + return x.Operation + } + return "" +} + +func (x *ModelCallView) GetProvider() string { + if x != nil { + return x.Provider + } + return "" +} + +func (x *ModelCallView) GetRequestModel() string { + if x != nil { + return x.RequestModel + } + return "" +} + +func (x *ModelCallView) GetResponseModel() string { + if x != nil { + return x.ResponseModel + } + return "" +} + +func (x *ModelCallView) GetResponseId() string { + if x != nil { + return x.ResponseId + } + return "" +} + +func (x *ModelCallView) GetUsage() *Usage { + if x != nil { + return x.Usage + } + return nil +} + +func (x *ModelCallView) GetFinishReasons() []string { + if x != nil { + return x.FinishReasons + } + return nil +} + +func (x *ModelCallView) GetStatus() v1.Status_StatusCode { + if x != nil { + return x.Status + } + return v1.Status_StatusCode(0) +} + +func (x *ModelCallView) GetErrorType() string { + if x != nil { + return x.ErrorType + } + return "" +} + +func (x *ModelCallView) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *ModelCallView) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +func (x *ModelCallView) GetAgentName() string { + if x != nil { + return x.AgentName + } + return "" +} + +type TurnView struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Position in the session or conversation, from 0. + Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` + // The turn's anchor: an agent or workflow span, or the trace root. + TraceId string `protobuf:"bytes,2,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanId string `protobuf:"bytes,3,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + // The anchor's span status. + Status v1.Status_StatusCode `protobuf:"varint,4,opt,name=status,proto3,enum=opentelemetry.proto.trace.v1.Status_StatusCode" json:"status,omitempty"` + // error.type. + ErrorType string `protobuf:"bytes,5,opt,name=error_type,json=errorType,proto3" json:"error_type,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + Duration *durationpb.Duration `protobuf:"bytes,7,opt,name=duration,proto3" json:"duration,omitempty"` + // gen_ai.agent.name of the anchor. + AgentName string `protobuf:"bytes,8,opt,name=agent_name,json=agentName,proto3" json:"agent_name,omitempty"` + // The user message that started the turn (gen_ai.input.messages). Unset when not recorded. + UserText *string `protobuf:"bytes,9,opt,name=user_text,json=userText,proto3,oneof" json:"user_text,omitempty"` + // The final answer (gen_ai.output.messages). Unset when not recorded. + AnswerText *string `protobuf:"bytes,10,opt,name=answer_text,json=answerText,proto3,oneof" json:"answer_text,omitempty"` + // Whether message content was recorded for this turn. + ContentCaptured bool `protobuf:"varint,11,opt,name=content_captured,json=contentCaptured,proto3" json:"content_captured,omitempty"` + ToolCalls []*ToolCallView `protobuf:"bytes,12,rep,name=tool_calls,json=toolCalls,proto3" json:"tool_calls,omitempty"` + ModelCalls []*ModelCallView `protobuf:"bytes,13,rep,name=model_calls,json=modelCalls,proto3" json:"model_calls,omitempty"` + // Sum over model_calls. + Usage *Usage `protobuf:"bytes,14,opt,name=usage,proto3" json:"usage,omitempty"` + // gen_ai.evaluation.result events other producers attached to this turn's spans. + ExternalEvaluations []*EvaluationResult `protobuf:"bytes,15,rep,name=external_evaluations,json=externalEvaluations,proto3" json:"external_evaluations,omitempty"` + Gaps []*Gap `protobuf:"bytes,16,rep,name=gaps,proto3" json:"gaps,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TurnView) Reset() { + *x = TurnView{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TurnView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TurnView) ProtoMessage() {} + +func (x *TurnView) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TurnView.ProtoReflect.Descriptor instead. +func (*TurnView) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{5} +} + +func (x *TurnView) GetIndex() int32 { + if x != nil { + return x.Index + } + return 0 +} + +func (x *TurnView) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +func (x *TurnView) GetSpanId() string { + if x != nil { + return x.SpanId + } + return "" +} + +func (x *TurnView) GetStatus() v1.Status_StatusCode { + if x != nil { + return x.Status + } + return v1.Status_StatusCode(0) +} + +func (x *TurnView) GetErrorType() string { + if x != nil { + return x.ErrorType + } + return "" +} + +func (x *TurnView) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *TurnView) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +func (x *TurnView) GetAgentName() string { + if x != nil { + return x.AgentName + } + return "" +} + +func (x *TurnView) GetUserText() string { + if x != nil && x.UserText != nil { + return *x.UserText + } + return "" +} + +func (x *TurnView) GetAnswerText() string { + if x != nil && x.AnswerText != nil { + return *x.AnswerText + } + return "" +} + +func (x *TurnView) GetContentCaptured() bool { + if x != nil { + return x.ContentCaptured + } + return false +} + +func (x *TurnView) GetToolCalls() []*ToolCallView { + if x != nil { + return x.ToolCalls + } + return nil +} + +func (x *TurnView) GetModelCalls() []*ModelCallView { + if x != nil { + return x.ModelCalls + } + return nil +} + +func (x *TurnView) GetUsage() *Usage { + if x != nil { + return x.Usage + } + return nil +} + +func (x *TurnView) GetExternalEvaluations() []*EvaluationResult { + if x != nil { + return x.ExternalEvaluations + } + return nil +} + +func (x *TurnView) GetGaps() []*Gap { + if x != nil { + return x.Gaps + } + return nil +} + +type SessionTotals struct { + state protoimpl.MessageState `protogen:"open.v1"` + Turns int32 `protobuf:"varint,1,opt,name=turns,proto3" json:"turns,omitempty"` + // Turns whose status is ERROR. + FailedTurns int32 `protobuf:"varint,2,opt,name=failed_turns,json=failedTurns,proto3" json:"failed_turns,omitempty"` + ModelCalls int32 `protobuf:"varint,3,opt,name=model_calls,json=modelCalls,proto3" json:"model_calls,omitempty"` + ToolCalls int32 `protobuf:"varint,4,opt,name=tool_calls,json=toolCalls,proto3" json:"tool_calls,omitempty"` + // Sum over all model calls. + Usage *Usage `protobuf:"bytes,5,opt,name=usage,proto3" json:"usage,omitempty"` + // Model calls that recorded token usage, for "measured in N of M". + ModelCallsWithUsage int32 `protobuf:"varint,6,opt,name=model_calls_with_usage,json=modelCallsWithUsage,proto3" json:"model_calls_with_usage,omitempty"` + // Tool calls that recorded arguments. + ToolCallsWithArguments int32 `protobuf:"varint,7,opt,name=tool_calls_with_arguments,json=toolCallsWithArguments,proto3" json:"tool_calls_with_arguments,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionTotals) Reset() { + *x = SessionTotals{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionTotals) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionTotals) ProtoMessage() {} + +func (x *SessionTotals) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionTotals.ProtoReflect.Descriptor instead. +func (*SessionTotals) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{6} +} + +func (x *SessionTotals) GetTurns() int32 { + if x != nil { + return x.Turns + } + return 0 +} + +func (x *SessionTotals) GetFailedTurns() int32 { + if x != nil { + return x.FailedTurns + } + return 0 +} + +func (x *SessionTotals) GetModelCalls() int32 { + if x != nil { + return x.ModelCalls + } + return 0 +} + +func (x *SessionTotals) GetToolCalls() int32 { + if x != nil { + return x.ToolCalls + } + return 0 +} + +func (x *SessionTotals) GetUsage() *Usage { + if x != nil { + return x.Usage + } + return nil +} + +func (x *SessionTotals) GetModelCallsWithUsage() int32 { + if x != nil { + return x.ModelCallsWithUsage + } + return 0 +} + +func (x *SessionTotals) GetToolCallsWithArguments() int32 { + if x != nil { + return x.ToolCallsWithArguments + } + return 0 +} + +type SessionSummary struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The last 6 characters of a UUID or hex id, which are often time ordered so their first + // characters collide; any other id, such as a session name, as it is. + ShortId string `protobuf:"bytes,2,opt,name=short_id,json=shortId,proto3" json:"short_id,omitempty"` + // The resource's gen_ai.agent.name, else the first turn's agent, else the resource's + // service.name. + Agent string `protobuf:"bytes,3,opt,name=agent,proto3" json:"agent,omitempty"` + // The resource's gen_ai.agent.version, else the first turn's, else service.version. + AgentVersion string `protobuf:"bytes,4,opt,name=agent_version,json=agentVersion,proto3" json:"agent_version,omitempty"` + State SessionState `protobuf:"varint,5,opt,name=state,proto3,enum=agentevals.api.v1alpha1.SessionState" json:"state,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + EndedAt *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` + Duration *durationpb.Duration `protobuf:"bytes,8,opt,name=duration,proto3" json:"duration,omitempty"` + // The first turn's user text. Unset when not recorded. + FirstUserText *string `protobuf:"bytes,9,opt,name=first_user_text,json=firstUserText,proto3,oneof" json:"first_user_text,omitempty"` + ContentCapture ContentCapture `protobuf:"varint,10,opt,name=content_capture,json=contentCapture,proto3,enum=agentevals.api.v1alpha1.ContentCapture" json:"content_capture,omitempty"` + Totals *SessionTotals `protobuf:"bytes,11,opt,name=totals,proto3" json:"totals,omitempty"` + // gen_ai.response.model of each call, else gen_ai.request.model, distinct, in first use + // order. + Models []string `protobuf:"bytes,12,rep,name=models,proto3" json:"models,omitempty"` + // The resource of the service that produced the session. + Resource *v11.Resource `protobuf:"bytes,13,opt,name=resource,proto3" json:"resource,omitempty"` + // gen_ai.conversation.id. + ConversationId string `protobuf:"bytes,14,opt,name=conversation_id,json=conversationId,proto3" json:"conversation_id,omitempty"` + TraceIds []string `protobuf:"bytes,15,rep,name=trace_ids,json=traceIds,proto3" json:"trace_ids,omitempty"` + // The eval set the session was recorded for, when the producer named one. + EvalSetId string `protobuf:"bytes,16,opt,name=eval_set_id,json=evalSetId,proto3" json:"eval_set_id,omitempty"` + SpanCount int32 `protobuf:"varint,17,opt,name=span_count,json=spanCount,proto3" json:"span_count,omitempty"` + Gaps []*Gap `protobuf:"bytes,18,rep,name=gaps,proto3" json:"gaps,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionSummary) Reset() { + *x = SessionSummary{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionSummary) ProtoMessage() {} + +func (x *SessionSummary) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionSummary.ProtoReflect.Descriptor instead. +func (*SessionSummary) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{7} +} + +func (x *SessionSummary) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SessionSummary) GetShortId() string { + if x != nil { + return x.ShortId + } + return "" +} + +func (x *SessionSummary) GetAgent() string { + if x != nil { + return x.Agent + } + return "" +} + +func (x *SessionSummary) GetAgentVersion() string { + if x != nil { + return x.AgentVersion + } + return "" +} + +func (x *SessionSummary) GetState() SessionState { + if x != nil { + return x.State + } + return SessionState_SESSION_STATE_UNSPECIFIED +} + +func (x *SessionSummary) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *SessionSummary) GetEndedAt() *timestamppb.Timestamp { + if x != nil { + return x.EndedAt + } + return nil +} + +func (x *SessionSummary) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +func (x *SessionSummary) GetFirstUserText() string { + if x != nil && x.FirstUserText != nil { + return *x.FirstUserText + } + return "" +} + +func (x *SessionSummary) GetContentCapture() ContentCapture { + if x != nil { + return x.ContentCapture + } + return ContentCapture_CONTENT_CAPTURE_UNSPECIFIED +} + +func (x *SessionSummary) GetTotals() *SessionTotals { + if x != nil { + return x.Totals + } + return nil +} + +func (x *SessionSummary) GetModels() []string { + if x != nil { + return x.Models + } + return nil +} + +func (x *SessionSummary) GetResource() *v11.Resource { + if x != nil { + return x.Resource + } + return nil +} + +func (x *SessionSummary) GetConversationId() string { + if x != nil { + return x.ConversationId + } + return "" +} + +func (x *SessionSummary) GetTraceIds() []string { + if x != nil { + return x.TraceIds + } + return nil +} + +func (x *SessionSummary) GetEvalSetId() string { + if x != nil { + return x.EvalSetId + } + return "" +} + +func (x *SessionSummary) GetSpanCount() int32 { + if x != nil { + return x.SpanCount + } + return 0 +} + +func (x *SessionSummary) GetGaps() []*Gap { + if x != nil { + return x.Gaps + } + return nil +} + +type Session struct { + state protoimpl.MessageState `protogen:"open.v1"` + Summary *SessionSummary `protobuf:"bytes,1,opt,name=summary,proto3" json:"summary,omitempty"` + Turns []*TurnView `protobuf:"bytes,2,rep,name=turns,proto3" json:"turns,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Session) Reset() { + *x = Session{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Session) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Session) ProtoMessage() {} + +func (x *Session) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Session.ProtoReflect.Descriptor instead. +func (*Session) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{8} +} + +func (x *Session) GetSummary() *SessionSummary { + if x != nil { + return x.Summary + } + return nil +} + +func (x *Session) GetTurns() []*TurnView { + if x != nil { + return x.Turns + } + return nil +} + +// Traces that form one evaluation unit, as conversation turns. +type ConversationView struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The unit's key: its earliest trace id, or its conversation key. + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // gen_ai.conversation.id. + ConversationId string `protobuf:"bytes,2,opt,name=conversation_id,json=conversationId,proto3" json:"conversation_id,omitempty"` + TraceIds []string `protobuf:"bytes,3,rep,name=trace_ids,json=traceIds,proto3" json:"trace_ids,omitempty"` + Turns []*TurnView `protobuf:"bytes,4,rep,name=turns,proto3" json:"turns,omitempty"` + Gaps []*Gap `protobuf:"bytes,5,rep,name=gaps,proto3" json:"gaps,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConversationView) Reset() { + *x = ConversationView{} + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConversationView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConversationView) ProtoMessage() {} + +func (x *ConversationView) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_views_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConversationView.ProtoReflect.Descriptor instead. +func (*ConversationView) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_views_proto_rawDescGZIP(), []int{9} +} + +func (x *ConversationView) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ConversationView) GetConversationId() string { + if x != nil { + return x.ConversationId + } + return "" +} + +func (x *ConversationView) GetTraceIds() []string { + if x != nil { + return x.TraceIds + } + return nil +} + +func (x *ConversationView) GetTurns() []*TurnView { + if x != nil { + return x.Turns + } + return nil +} + +func (x *ConversationView) GetGaps() []*Gap { + if x != nil { + return x.Gaps + } + return nil +} + +var File_agentevals_api_v1alpha1_views_proto protoreflect.FileDescriptor + +const file_agentevals_api_v1alpha1_views_proto_rawDesc = "" + + "\n" + + "#agentevals/api/v1alpha1/views.proto\x12\x17agentevals.api.v1alpha1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\"\x88\x03\n" + + "\x05Usage\x12&\n" + + "\finput_tokens\x18\x01 \x01(\x03H\x00R\vinputTokens\x88\x01\x01\x12(\n" + + "\routput_tokens\x18\x02 \x01(\x03H\x01R\foutputTokens\x88\x01\x01\x12:\n" + + "\x17cache_read_input_tokens\x18\x03 \x01(\x03H\x02R\x14cacheReadInputTokens\x88\x01\x01\x12<\n" + + "\x18cache_write_input_tokens\x18\x04 \x01(\x03H\x03R\x15cacheWriteInputTokens\x88\x01\x01\x12;\n" + + "\x17reasoning_output_tokens\x18\x05 \x01(\x03H\x04R\x15reasoningOutputTokens\x88\x01\x01B\x0f\n" + + "\r_input_tokensB\x10\n" + + "\x0e_output_tokensB\x1a\n" + + "\x18_cache_read_input_tokensB\x1b\n" + + "\x19_cache_write_input_tokensB\x1a\n" + + "\x18_reasoning_output_tokens\"1\n" + + "\x03Gap\x12\x12\n" + + "\x04code\x18\x01 \x01(\tR\x04code\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\"\xf8\x02\n" + + "\x10EvaluationResult\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12$\n" + + "\vscore_value\x18\x02 \x01(\x01H\x00R\n" + + "scoreValue\x88\x01\x01\x12$\n" + + "\vscore_label\x18\x03 \x01(\tH\x01R\n" + + "scoreLabel\x88\x01\x01\x12%\n" + + "\vexplanation\x18\x04 \x01(\tH\x02R\vexplanation\x88\x01\x01\x12\"\n" + + "\n" + + "error_type\x18\x05 \x01(\tH\x03R\terrorType\x88\x01\x01\x12\x19\n" + + "\btrace_id\x18\x06 \x01(\tR\atraceId\x12\x17\n" + + "\aspan_id\x18\a \x01(\tR\x06spanId\x12\x1f\n" + + "\vresponse_id\x18\b \x01(\tR\n" + + "responseId\x12%\n" + + "\x0esource_service\x18\t \x01(\tR\rsourceServiceB\x0e\n" + + "\f_score_valueB\x0e\n" + + "\f_score_labelB\x0e\n" + + "\f_explanationB\r\n" + + "\v_error_type\"\xf7\x02\n" + + "\fToolCallView\x12\x17\n" + + "\aspan_id\x18\x01 \x01(\tR\x06spanId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1c\n" + + "\acall_id\x18\x03 \x01(\tH\x00R\x06callId\x88\x01\x01\x124\n" + + "\targuments\x18\x04 \x01(\v2\x16.google.protobuf.ValueR\targuments\x12.\n" + + "\x06result\x18\x05 \x01(\v2\x16.google.protobuf.ValueR\x06result\x12\x19\n" + + "\bis_error\x18\x06 \x01(\bR\aisError\x12\x1d\n" + + "\n" + + "error_type\x18\a \x01(\tR\terrorType\x129\n" + + "\n" + + "started_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bduration\x18\t \x01(\v2\x19.google.protobuf.DurationR\bdurationB\n" + + "\n" + + "\b_call_id\"\xa5\x04\n" + + "\rModelCallView\x12\x17\n" + + "\aspan_id\x18\x01 \x01(\tR\x06spanId\x12\x1c\n" + + "\toperation\x18\x02 \x01(\tR\toperation\x12\x1a\n" + + "\bprovider\x18\x03 \x01(\tR\bprovider\x12#\n" + + "\rrequest_model\x18\x04 \x01(\tR\frequestModel\x12%\n" + + "\x0eresponse_model\x18\x05 \x01(\tR\rresponseModel\x12\x1f\n" + + "\vresponse_id\x18\x06 \x01(\tR\n" + + "responseId\x124\n" + + "\x05usage\x18\a \x01(\v2\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x12%\n" + + "\x0efinish_reasons\x18\b \x03(\tR\rfinishReasons\x12G\n" + + "\x06status\x18\t \x01(\x0e2/.opentelemetry.proto.trace.v1.Status.StatusCodeR\x06status\x12\x1d\n" + + "\n" + + "error_type\x18\n" + + " \x01(\tR\terrorType\x129\n" + + "\n" + + "started_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bduration\x18\f \x01(\v2\x19.google.protobuf.DurationR\bduration\x12\x1d\n" + + "\n" + + "agent_name\x18\r \x01(\tR\tagentName\"\xb3\x06\n" + + "\bTurnView\x12\x14\n" + + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x19\n" + + "\btrace_id\x18\x02 \x01(\tR\atraceId\x12\x17\n" + + "\aspan_id\x18\x03 \x01(\tR\x06spanId\x12G\n" + + "\x06status\x18\x04 \x01(\x0e2/.opentelemetry.proto.trace.v1.Status.StatusCodeR\x06status\x12\x1d\n" + + "\n" + + "error_type\x18\x05 \x01(\tR\terrorType\x129\n" + + "\n" + + "started_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bduration\x18\a \x01(\v2\x19.google.protobuf.DurationR\bduration\x12\x1d\n" + + "\n" + + "agent_name\x18\b \x01(\tR\tagentName\x12 \n" + + "\tuser_text\x18\t \x01(\tH\x00R\buserText\x88\x01\x01\x12$\n" + + "\vanswer_text\x18\n" + + " \x01(\tH\x01R\n" + + "answerText\x88\x01\x01\x12)\n" + + "\x10content_captured\x18\v \x01(\bR\x0fcontentCaptured\x12D\n" + + "\n" + + "tool_calls\x18\f \x03(\v2%.agentevals.api.v1alpha1.ToolCallViewR\ttoolCalls\x12G\n" + + "\vmodel_calls\x18\r \x03(\v2&.agentevals.api.v1alpha1.ModelCallViewR\n" + + "modelCalls\x124\n" + + "\x05usage\x18\x0e \x01(\v2\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x12\\\n" + + "\x14external_evaluations\x18\x0f \x03(\v2).agentevals.api.v1alpha1.EvaluationResultR\x13externalEvaluations\x120\n" + + "\x04gaps\x18\x10 \x03(\v2\x1c.agentevals.api.v1alpha1.GapR\x04gapsB\f\n" + + "\n" + + "_user_textB\x0e\n" + + "\f_answer_text\"\xae\x02\n" + + "\rSessionTotals\x12\x14\n" + + "\x05turns\x18\x01 \x01(\x05R\x05turns\x12!\n" + + "\ffailed_turns\x18\x02 \x01(\x05R\vfailedTurns\x12\x1f\n" + + "\vmodel_calls\x18\x03 \x01(\x05R\n" + + "modelCalls\x12\x1d\n" + + "\n" + + "tool_calls\x18\x04 \x01(\x05R\ttoolCalls\x124\n" + + "\x05usage\x18\x05 \x01(\v2\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x123\n" + + "\x16model_calls_with_usage\x18\x06 \x01(\x05R\x13modelCallsWithUsage\x129\n" + + "\x19tool_calls_with_arguments\x18\a \x01(\x05R\x16toolCallsWithArguments\"\xc5\x06\n" + + "\x0eSessionSummary\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x19\n" + + "\bshort_id\x18\x02 \x01(\tR\ashortId\x12\x14\n" + + "\x05agent\x18\x03 \x01(\tR\x05agent\x12#\n" + + "\ragent_version\x18\x04 \x01(\tR\fagentVersion\x12;\n" + + "\x05state\x18\x05 \x01(\x0e2%.agentevals.api.v1alpha1.SessionStateR\x05state\x129\n" + + "\n" + + "started_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bended_at\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x125\n" + + "\bduration\x18\b \x01(\v2\x19.google.protobuf.DurationR\bduration\x12+\n" + + "\x0ffirst_user_text\x18\t \x01(\tH\x00R\rfirstUserText\x88\x01\x01\x12P\n" + + "\x0fcontent_capture\x18\n" + + " \x01(\x0e2'.agentevals.api.v1alpha1.ContentCaptureR\x0econtentCapture\x12>\n" + + "\x06totals\x18\v \x01(\v2&.agentevals.api.v1alpha1.SessionTotalsR\x06totals\x12\x16\n" + + "\x06models\x18\f \x03(\tR\x06models\x12E\n" + + "\bresource\x18\r \x01(\v2).opentelemetry.proto.resource.v1.ResourceR\bresource\x12'\n" + + "\x0fconversation_id\x18\x0e \x01(\tR\x0econversationId\x12\x1b\n" + + "\ttrace_ids\x18\x0f \x03(\tR\btraceIds\x12\x1e\n" + + "\veval_set_id\x18\x10 \x01(\tR\tevalSetId\x12\x1d\n" + + "\n" + + "span_count\x18\x11 \x01(\x05R\tspanCount\x120\n" + + "\x04gaps\x18\x12 \x03(\v2\x1c.agentevals.api.v1alpha1.GapR\x04gapsB\x12\n" + + "\x10_first_user_text\"\x85\x01\n" + + "\aSession\x12A\n" + + "\asummary\x18\x01 \x01(\v2'.agentevals.api.v1alpha1.SessionSummaryR\asummary\x127\n" + + "\x05turns\x18\x02 \x03(\v2!.agentevals.api.v1alpha1.TurnViewR\x05turns\"\xd5\x01\n" + + "\x10ConversationView\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12'\n" + + "\x0fconversation_id\x18\x02 \x01(\tR\x0econversationId\x12\x1b\n" + + "\ttrace_ids\x18\x03 \x03(\tR\btraceIds\x127\n" + + "\x05turns\x18\x04 \x03(\v2!.agentevals.api.v1alpha1.TurnViewR\x05turns\x120\n" + + "\x04gaps\x18\x05 \x03(\v2\x1c.agentevals.api.v1alpha1.GapR\x04gaps*a\n" + + "\fSessionState\x12\x1d\n" + + "\x19SESSION_STATE_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12SESSION_STATE_LIVE\x10\x01\x12\x1a\n" + + "\x16SESSION_STATE_COMPLETE\x10\x02*\x7f\n" + + "\x0eContentCapture\x12\x1f\n" + + "\x1bCONTENT_CAPTURE_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12CONTENT_CAPTURE_ON\x10\x01\x12\x17\n" + + "\x13CONTENT_CAPTURE_OFF\x10\x02\x12\x1b\n" + + "\x17CONTENT_CAPTURE_PARTIAL\x10\x03BQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3" + +var ( + file_agentevals_api_v1alpha1_views_proto_rawDescOnce sync.Once + file_agentevals_api_v1alpha1_views_proto_rawDescData []byte +) + +func file_agentevals_api_v1alpha1_views_proto_rawDescGZIP() []byte { + file_agentevals_api_v1alpha1_views_proto_rawDescOnce.Do(func() { + file_agentevals_api_v1alpha1_views_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_views_proto_rawDesc), len(file_agentevals_api_v1alpha1_views_proto_rawDesc))) + }) + return file_agentevals_api_v1alpha1_views_proto_rawDescData +} + +var file_agentevals_api_v1alpha1_views_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_agentevals_api_v1alpha1_views_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_agentevals_api_v1alpha1_views_proto_goTypes = []any{ + (SessionState)(0), // 0: agentevals.api.v1alpha1.SessionState + (ContentCapture)(0), // 1: agentevals.api.v1alpha1.ContentCapture + (*Usage)(nil), // 2: agentevals.api.v1alpha1.Usage + (*Gap)(nil), // 3: agentevals.api.v1alpha1.Gap + (*EvaluationResult)(nil), // 4: agentevals.api.v1alpha1.EvaluationResult + (*ToolCallView)(nil), // 5: agentevals.api.v1alpha1.ToolCallView + (*ModelCallView)(nil), // 6: agentevals.api.v1alpha1.ModelCallView + (*TurnView)(nil), // 7: agentevals.api.v1alpha1.TurnView + (*SessionTotals)(nil), // 8: agentevals.api.v1alpha1.SessionTotals + (*SessionSummary)(nil), // 9: agentevals.api.v1alpha1.SessionSummary + (*Session)(nil), // 10: agentevals.api.v1alpha1.Session + (*ConversationView)(nil), // 11: agentevals.api.v1alpha1.ConversationView + (*structpb.Value)(nil), // 12: google.protobuf.Value + (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 14: google.protobuf.Duration + (v1.Status_StatusCode)(0), // 15: opentelemetry.proto.trace.v1.Status.StatusCode + (*v11.Resource)(nil), // 16: opentelemetry.proto.resource.v1.Resource +} +var file_agentevals_api_v1alpha1_views_proto_depIdxs = []int32{ + 12, // 0: agentevals.api.v1alpha1.ToolCallView.arguments:type_name -> google.protobuf.Value + 12, // 1: agentevals.api.v1alpha1.ToolCallView.result:type_name -> google.protobuf.Value + 13, // 2: agentevals.api.v1alpha1.ToolCallView.started_at:type_name -> google.protobuf.Timestamp + 14, // 3: agentevals.api.v1alpha1.ToolCallView.duration:type_name -> google.protobuf.Duration + 2, // 4: agentevals.api.v1alpha1.ModelCallView.usage:type_name -> agentevals.api.v1alpha1.Usage + 15, // 5: agentevals.api.v1alpha1.ModelCallView.status:type_name -> opentelemetry.proto.trace.v1.Status.StatusCode + 13, // 6: agentevals.api.v1alpha1.ModelCallView.started_at:type_name -> google.protobuf.Timestamp + 14, // 7: agentevals.api.v1alpha1.ModelCallView.duration:type_name -> google.protobuf.Duration + 15, // 8: agentevals.api.v1alpha1.TurnView.status:type_name -> opentelemetry.proto.trace.v1.Status.StatusCode + 13, // 9: agentevals.api.v1alpha1.TurnView.started_at:type_name -> google.protobuf.Timestamp + 14, // 10: agentevals.api.v1alpha1.TurnView.duration:type_name -> google.protobuf.Duration + 5, // 11: agentevals.api.v1alpha1.TurnView.tool_calls:type_name -> agentevals.api.v1alpha1.ToolCallView + 6, // 12: agentevals.api.v1alpha1.TurnView.model_calls:type_name -> agentevals.api.v1alpha1.ModelCallView + 2, // 13: agentevals.api.v1alpha1.TurnView.usage:type_name -> agentevals.api.v1alpha1.Usage + 4, // 14: agentevals.api.v1alpha1.TurnView.external_evaluations:type_name -> agentevals.api.v1alpha1.EvaluationResult + 3, // 15: agentevals.api.v1alpha1.TurnView.gaps:type_name -> agentevals.api.v1alpha1.Gap + 2, // 16: agentevals.api.v1alpha1.SessionTotals.usage:type_name -> agentevals.api.v1alpha1.Usage + 0, // 17: agentevals.api.v1alpha1.SessionSummary.state:type_name -> agentevals.api.v1alpha1.SessionState + 13, // 18: agentevals.api.v1alpha1.SessionSummary.started_at:type_name -> google.protobuf.Timestamp + 13, // 19: agentevals.api.v1alpha1.SessionSummary.ended_at:type_name -> google.protobuf.Timestamp + 14, // 20: agentevals.api.v1alpha1.SessionSummary.duration:type_name -> google.protobuf.Duration + 1, // 21: agentevals.api.v1alpha1.SessionSummary.content_capture:type_name -> agentevals.api.v1alpha1.ContentCapture + 8, // 22: agentevals.api.v1alpha1.SessionSummary.totals:type_name -> agentevals.api.v1alpha1.SessionTotals + 16, // 23: agentevals.api.v1alpha1.SessionSummary.resource:type_name -> opentelemetry.proto.resource.v1.Resource + 3, // 24: agentevals.api.v1alpha1.SessionSummary.gaps:type_name -> agentevals.api.v1alpha1.Gap + 9, // 25: agentevals.api.v1alpha1.Session.summary:type_name -> agentevals.api.v1alpha1.SessionSummary + 7, // 26: agentevals.api.v1alpha1.Session.turns:type_name -> agentevals.api.v1alpha1.TurnView + 7, // 27: agentevals.api.v1alpha1.ConversationView.turns:type_name -> agentevals.api.v1alpha1.TurnView + 3, // 28: agentevals.api.v1alpha1.ConversationView.gaps:type_name -> agentevals.api.v1alpha1.Gap + 29, // [29:29] is the sub-list for method output_type + 29, // [29:29] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name +} + +func init() { file_agentevals_api_v1alpha1_views_proto_init() } +func file_agentevals_api_v1alpha1_views_proto_init() { + if File_agentevals_api_v1alpha1_views_proto != nil { + return + } + file_agentevals_api_v1alpha1_views_proto_msgTypes[0].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_views_proto_msgTypes[2].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_views_proto_msgTypes[3].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_views_proto_msgTypes[5].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_views_proto_msgTypes[7].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_views_proto_rawDesc), len(file_agentevals_api_v1alpha1_views_proto_rawDesc)), + NumEnums: 2, + NumMessages: 10, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_agentevals_api_v1alpha1_views_proto_goTypes, + DependencyIndexes: file_agentevals_api_v1alpha1_views_proto_depIdxs, + EnumInfos: file_agentevals_api_v1alpha1_views_proto_enumTypes, + MessageInfos: file_agentevals_api_v1alpha1_views_proto_msgTypes, + }.Build() + File_agentevals_api_v1alpha1_views_proto = out.File + file_agentevals_api_v1alpha1_views_proto_goTypes = nil + file_agentevals_api_v1alpha1_views_proto_depIdxs = nil +} diff --git a/gen/go/go.mod b/gen/go/go.mod new file mode 100644 index 00000000..f3e77c85 --- /dev/null +++ b/gen/go/go.mod @@ -0,0 +1,16 @@ +module github.com/agentevals-dev/agentevals/gen/go + +go 1.24 + +require ( + go.opentelemetry.io/proto/otlp v1.9.0 + google.golang.org/grpc v1.75.1 + google.golang.org/protobuf v1.36.11 +) + +require ( + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect +) diff --git a/gen/go/go.sum b/gen/go/go.sum new file mode 100644 index 00000000..c86a4c83 --- /dev/null +++ b/gen/go/go.sum @@ -0,0 +1,38 @@ +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/proto/agentevals/api/v1alpha1/common.proto b/proto/agentevals/api/v1alpha1/common.proto new file mode 100644 index 00000000..1b8649f7 --- /dev/null +++ b/proto/agentevals/api/v1alpha1/common.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "opentelemetry/proto/logs/v1/logs.proto"; +import "opentelemetry/proto/trace/v1/trace.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +// limit 0 means the default of 50; the maximum is 100. +message PageRequest { + int32 limit = 1; + // Opaque token from a previous PageResponse. + string page_token = 2; +} + +message PageResponse { + // Empty on the last page. + string next_page_token = 1; +} + +enum TraceFormat { + // Detected from the content. + TRACE_FORMAT_UNSPECIFIED = 0; + // One OTLP/JSON document (hex ids, as the OTLP/HTTP JSON encoding). + TRACE_FORMAT_OTLP_JSON = 1; + // One OTLP/JSON document or span per line. + TRACE_FORMAT_OTLP_JSONL = 2; + // A Jaeger JSON export. + TRACE_FORMAT_JAEGER_JSON = 3; +} + +// A trace file as bytes, as the REST API takes uploads. At most 10 MiB. +message TraceDocument { + bytes content = 1; + TraceFormat format = 2; + // The file name, used in messages and as a session name hint (trace_.json). + string name = 3; +} + +// OTLP telemetry as protobuf. Logs carry the GenAI events that belong to the spans. +message OtlpTelemetry { + opentelemetry.proto.trace.v1.TracesData traces = 1; + opentelemetry.proto.logs.v1.LogsData logs = 2; +} + +// Traces to read. Both fields may be set; their traces are combined. +message TraceInput { + OtlpTelemetry otlp = 1; + repeated TraceDocument documents = 2; +} + +// How traces form evaluation units. +enum GroupBy { + // The default of the field that uses it. + GROUP_BY_UNSPECIFIED = 0; + // Conversation when any trace carries agentevals.session_name or the eval set has a multi + // turn case, otherwise trace. + GROUP_BY_AUTO = 1; + // Every trace on its own. + GROUP_BY_TRACE = 2; + // Traces sharing agentevals.session_name, gen_ai.conversation.id or session.id form one unit. + GROUP_BY_CONVERSATION = 3; +} diff --git a/proto/agentevals/api/v1alpha1/conversion.proto b/proto/agentevals/api/v1alpha1/conversion.proto new file mode 100644 index 00000000..9d86ccb5 --- /dev/null +++ b/proto/agentevals/api/v1alpha1/conversion.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "agentevals/api/v1alpha1/common.proto"; +import "agentevals/api/v1alpha1/views.proto"; +import "google/protobuf/struct.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +service ConversionService { + // Reads traces into conversations without evaluating them. + rpc ConvertTraces(ConvertTracesRequest) returns (ConvertTracesResponse); +} + +message ConvertTracesRequest { + TraceInput traces = 1; + // Unset means GROUP_BY_TRACE. + GroupBy group_by = 2; +} + +message ConvertedConversation { + ConversationView conversation = 1; + // The conversation as an ADK EvalSet with one case, to use as a golden. Without captured + // content (see TurnView.content_captured) its user turns are empty. + google.protobuf.Struct golden_adk_eval_set = 2; +} + +message ConvertTracesResponse { + repeated ConvertedConversation conversations = 1; + // Documents that could not be read. + repeated string warnings = 2; +} diff --git a/proto/agentevals/api/v1alpha1/evaluation.proto b/proto/agentevals/api/v1alpha1/evaluation.proto new file mode 100644 index 00000000..ebfb98c5 --- /dev/null +++ b/proto/agentevals/api/v1alpha1/evaluation.proto @@ -0,0 +1,210 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "agentevals/api/v1alpha1/common.proto"; +import "google/protobuf/struct.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +service EvaluationService { + rpc ListMetrics(ListMetricsRequest) returns (ListMetricsResponse); + // Evaluates traces. Streams progress, each group's performance metrics before evaluation, each + // group's results, and the final result, after which the stream ends. When run storage is + // configured, the evaluation is also recorded as a run. + rpc EvaluateTraces(EvaluateTracesRequest) returns (stream EvaluateTracesResponse); +} + +message ListMetricsRequest {} + +message Metric { + string name = 1; + string category = 2; + string description = 3; + // Needs a golden eval set. + bool requires_eval_set = 4; + // Calls a judge model. + bool requires_llm = 5; + // Needs Google Cloud credentials. + bool requires_gcp = 6; + // Needs rubrics in the eval set. + bool requires_rubrics = 7; + // Usable with recorded traces. + bool working = 8; +} + +message ListMetricsResponse { + repeated Metric metrics = 1; +} + +enum TrajectoryMatchType { + // The metric's default. + TRAJECTORY_MATCH_TYPE_UNSPECIFIED = 0; + TRAJECTORY_MATCH_TYPE_EXACT = 1; + TRAJECTORY_MATCH_TYPE_IN_ORDER = 2; + TRAJECTORY_MATCH_TYPE_ANY_ORDER = 3; +} + +// A built-in metric, by name (see ListMetrics), with optional overrides. +message BuiltinEvaluator { + string name = 1; + // 0 to 1. Unset uses the metric's default. + optional double threshold = 2; + string judge_model = 3; + TrajectoryMatchType trajectory_match_type = 4; + // Name of a credential_refs entry whose value is the judge API key. + string credential_ref = 5; + // Base URL of an OpenAI compatible judge endpoint. + string judge_base_url = 6; +} + +// An evaluator file on the server, run as a subprocess. +message CodeEvaluator { + string name = 1; + // 0 to 1. Unset means 0.5. + optional double threshold = 2; + // Seconds. Unset means 30. + optional int32 timeout = 3; + google.protobuf.Struct config = 4; + // Unset means local. + string executor = 5; + // Path of the evaluator file (.py, .js, .ts and so on) on the server. + string path = 6; +} + +// An evaluator fetched from a remote source. +message RemoteEvaluator { + string name = 1; + // 0 to 1. Unset means 0.5. + optional double threshold = 2; + // Seconds. Unset means 30. + optional int32 timeout = 3; + google.protobuf.Struct config = 4; + // Unset means local. + string executor = 5; + // Unset means github. + string source = 6; + // A relative path within the source. + string ref = 7; +} + +message Evaluator { + oneof kind { + BuiltinEvaluator builtin = 1; + CodeEvaluator code = 2; + RemoteEvaluator remote = 3; + } +} + +message EvalParams { + // Names must be unique. Empty means tool_trajectory_avg_score. + repeated Evaluator evaluators = 1; + // At least 1. Unset means 10. + optional int32 max_concurrent_traces = 2; + // At least 1. Unset means 5. + optional int32 max_concurrent_evals = 3; + // Unset means GROUP_BY_AUTO. + GroupBy group_by = 4; +} + +enum EvalStatus { + EVAL_STATUS_UNSPECIFIED = 0; + EVAL_STATUS_PASSED = 1; + EVAL_STATUS_FAILED = 2; + EVAL_STATUS_NOT_EVALUATED = 3; +} + +// A metric's result for one turn. +message InvocationResult { + // Unset when the turn has no score. + optional double score = 1; + EvalStatus status = 2; +} + +message MetricResult { + string metric_name = 1; + optional double score = 2; + EvalStatus eval_status = 3; + // One entry per turn, in turn order. + repeated InvocationResult per_invocation = 4; + optional string error = 5; + // error.type. + optional string error_type = 6; + // Metric specific details. + google.protobuf.Struct details = 7; + optional double duration_ms = 8; +} + +message TurnRef { + string trace_id = 1; + string span_id = 2; +} + +// The results of one evaluation group: a trace, or a conversation of several traces. +message TraceResult { + // The group's earliest trace. + string trace_id = 1; + int32 num_invocations = 2; + repeated MetricResult metric_results = 3; + repeated string conversion_warnings = 4; + google.protobuf.Struct performance_metrics = 5; + // service.name. + string service_name = 6; + // gen_ai.agent.name. + string agent_name = 7; + repeated string trace_ids = 8; + // gen_ai.conversation.id. + string conversation_id = 9; + string group_key = 10; + // The eval set case the group was compared with. + string eval_case_id = 11; + // How the case was chosen. + string eval_case_match = 12; + // The judged turns, in order. + repeated TurnRef turn_refs = 13; +} + +message RunResult { + repeated TraceResult trace_results = 1; + repeated string errors = 2; + google.protobuf.Struct performance_metrics = 3; + // Set when the evaluation was recorded as a run. + string run_id = 4; +} + +message EvaluateTracesRequest { + TraceInput traces = 1; + // The golden. Required by metrics with requires_eval_set. + oneof eval_set { + // An ADK EvalSet as JSON. + google.protobuf.Struct adk_eval_set = 2; + } + EvalParams params = 3; + // Logical credential name to a secret reference, for example + // {"kind": "kubernetes", "namespace": "...", "name": "...", "key": "..."} or + // {"kind": "env", "name": "..."}. Values are resolved per request and never returned. + map credential_refs = 4; +} + +message EvaluationProgress { + string message = 1; +} + +// A group's performance metrics, sent before it is evaluated. +message GroupPerformance { + // The group's earliest trace. + string trace_id = 1; + google.protobuf.Struct performance_metrics = 2; + google.protobuf.Struct trace_metadata = 3; +} + +message EvaluateTracesResponse { + oneof event { + EvaluationProgress progress = 1; + GroupPerformance performance = 2; + // A group's results so far. + TraceResult trace_result = 3; + // The final result. + RunResult done = 4; + } +} diff --git a/proto/agentevals/api/v1alpha1/runs.proto b/proto/agentevals/api/v1alpha1/runs.proto new file mode 100644 index 00000000..becffcca --- /dev/null +++ b/proto/agentevals/api/v1alpha1/runs.proto @@ -0,0 +1,201 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "agentevals/api/v1alpha1/common.proto"; +import "agentevals/api/v1alpha1/evaluation.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +// Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every +// method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. +service RunService { + // Queues a run. Idempotent on run_id: the same id with the same spec returns the existing run; + // a different spec fails with ALREADY_EXISTS and reason RUN_SPEC_CONFLICT. + rpc SubmitRun(SubmitRunRequest) returns (SubmitRunResponse); + rpc GetRun(GetRunRequest) returns (GetRunResponse); + // Newest first. + rpc ListRuns(ListRunsRequest) returns (ListRunsResponse); + // Requests cancellation. Cancelling a finished run changes nothing. + rpc CancelRun(CancelRunRequest) returns (CancelRunResponse); + rpc ListRunResults(ListRunResultsRequest) returns (ListRunResultsResponse); + // The run and its results, then each change, until the run reaches a terminal status. + rpc WatchRun(WatchRunRequest) returns (stream WatchRunResponse); +} + +enum RunStatus { + RUN_STATUS_UNSPECIFIED = 0; + RUN_STATUS_QUEUED = 1; + RUN_STATUS_RUNNING = 2; + RUN_STATUS_SUCCEEDED = 3; + RUN_STATUS_FAILED = 4; + RUN_STATUS_CANCELLED = 5; +} + +enum ResultStatus { + RESULT_STATUS_UNSPECIFIED = 0; + RESULT_STATUS_PASSED = 1; + RESULT_STATUS_FAILED = 2; + RESULT_STATUS_ERRORED = 3; + RESULT_STATUS_SKIPPED = 4; +} + +enum EvaluatorType { + EVALUATOR_TYPE_UNSPECIFIED = 0; + EVALUATOR_TYPE_BUILTIN = 1; + EVALUATOR_TYPE_CODE = 2; + EVALUATOR_TYPE_REMOTE = 3; +} + +enum RunApproach { + // RUN_APPROACH_TRACE_REPLAY. + RUN_APPROACH_UNSPECIFIED = 0; + // Evaluate recorded traces. + RUN_APPROACH_TRACE_REPLAY = 1; +} + +// A trace fetched with GET base_url/trace_id and the run's context headers. +message HttpTraceTarget { + string base_url = 1; + // 16 or 32 hex characters. + string trace_id = 2; + // TRACE_FORMAT_OTLP_JSON or TRACE_FORMAT_JAEGER_JSON. + TraceFormat format = 3; +} + +// A run recorded from a synchronous upload. The traces are not kept, so it cannot run again. +message UploadedTraces { + int32 trace_count = 1; + repeated string trace_files = 2; +} + +message TraceTarget { + oneof kind { + // A trace document in the spec, OTLP/JSON or Jaeger JSON. + TraceDocument inline = 1; + // OTLP protobuf. Stored, and returned, as an inline OTLP/JSON document. + OtlpTelemetry otlp = 2; + HttpTraceTarget http = 3; + // Output only. + UploadedTraces uploaded = 4; + } +} + +message RunContext { + // Headers for fetching an http target. Values are returned redacted. + map headers = 1; +} + +message RunSpec { + RunApproach approach = 1; + TraceTarget target = 2; + // The golden. + oneof eval_set { + // An ADK EvalSet as JSON. + google.protobuf.Struct adk_eval_set = 3; + } + EvalParams eval_config = 4; + // Result sinks, for example {"kind": "http_webhook", "url": "...", "headers": {...}}. Header + // values are returned redacted. + repeated google.protobuf.Struct sinks = 5; + RunContext context = 6; + // Logical credential name to a secret reference, as in EvaluateTracesRequest. + map credential_refs = 7; +} + +message Run { + // A UUID. + string run_id = 1; + RunStatus status = 2; + RunSpec spec = 3; + int32 attempt = 4; + string worker_id = 5; + string error = 6; + google.protobuf.Struct summary = 7; + google.protobuf.Timestamp created_at = 8; + google.protobuf.Timestamp started_at = 9; + google.protobuf.Timestamp finished_at = 10; + bool cancel_requested = 11; +} + +// One evaluator's result for one eval set item. +message Result { + // Stable: a SHA-256 of run id, item id and evaluator name. + string result_id = 1; + string run_id = 2; + string eval_set_item_id = 3; + string eval_set_item_name = 4; + string evaluator_name = 5; + EvaluatorType evaluator_type = 6; + ResultStatus status = 7; + optional double score = 8; + // One entry per turn, in turn order; status is not recorded. + repeated InvocationResult per_invocation = 9; + string trace_id = 10; + string span_id = 11; + google.protobuf.Struct details = 12; + string error_text = 13; + google.protobuf.Struct tokens_used = 14; + optional int64 latency_ms = 15; + google.protobuf.Timestamp created_at = 16; +} + +message SubmitRunRequest { + // A UUID chosen by the caller, for idempotent retries. Empty means the server picks one. + string run_id = 1; + RunSpec spec = 2; +} + +message SubmitRunResponse { + Run run = 1; +} + +message GetRunRequest { + string run_id = 1; +} + +message GetRunResponse { + Run run = 1; +} + +message ListRunsRequest { + // Empty means every status. + repeated RunStatus statuses = 1; + PageRequest page = 2; +} + +message ListRunsResponse { + repeated Run runs = 1; + PageResponse page = 2; +} + +message CancelRunRequest { + string run_id = 1; +} + +message CancelRunResponse { + Run run = 1; +} + +message ListRunResultsRequest { + string run_id = 1; + PageRequest page = 2; +} + +message ListRunResultsResponse { + repeated Result results = 1; + PageResponse page = 2; +} + +message WatchRunRequest { + string run_id = 1; +} + +message WatchRunResponse { + // The run, sent first and after every change. + Run run = 1; + // Results that are new or changed since the previous message. + repeated Result results = 2; +} diff --git a/proto/agentevals/api/v1alpha1/sessions.proto b/proto/agentevals/api/v1alpha1/sessions.proto new file mode 100644 index 00000000..b09995cb --- /dev/null +++ b/proto/agentevals/api/v1alpha1/sessions.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "agentevals/api/v1alpha1/common.proto"; +import "agentevals/api/v1alpha1/views.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +// Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in +// memory and expire. +service SessionService { + // Sessions newest first, summaries only. + rpc ListSessions(ListSessionsRequest) returns (ListSessionsResponse); + rpc GetSession(GetSessionRequest) returns (GetSessionResponse); + // Every current session, then synced, then each change as it happens. + rpc WatchSessions(WatchSessionsRequest) returns (stream WatchSessionsResponse); +} + +message ListSessionsRequest { + PageRequest page = 1; +} + +message ListSessionsResponse { + repeated SessionSummary sessions = 1; + PageResponse page = 2; +} + +message GetSessionRequest { + string id = 1; +} + +message GetSessionResponse { + Session session = 1; +} + +message WatchSessionsRequest { + // Send each session with its turns. Otherwise upserts carry the summary only. + bool include_turns = 1; +} + +message SessionRemoved { + string id = 1; + // The session that took over this session's traces, when the two were merged. + string absorbed_by = 2; +} + +// The end of the initial snapshot. +message Synced {} + +message WatchSessionsResponse { + oneof event { + // The session's current state. It replaces every earlier upsert with the same id. Upserts + // can be coalesced, so a client sees the latest state, not every intermediate one. + Session upsert = 1; + SessionRemoved removed = 2; + Synced synced = 3; + } +} diff --git a/proto/agentevals/api/v1alpha1/system.proto b/proto/agentevals/api/v1alpha1/system.proto new file mode 100644 index 00000000..543ad577 --- /dev/null +++ b/proto/agentevals/api/v1alpha1/system.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "google/protobuf/duration.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +service SystemService { + rpc GetVersion(GetVersionRequest) returns (GetVersionResponse); + rpc GetConfig(GetConfigRequest) returns (GetConfigResponse); +} + +message GetVersionRequest {} + +message GetVersionResponse { + string version = 1; +} + +message GetConfigRequest {} + +// Listener ports. 0 when a listener is off. +message Ports { + int32 http = 1; + int32 otlp_http = 2; + int32 otlp_grpc = 3; + int32 api_grpc = 4; +} + +// Bounds of the live session store. Telemetry over a bound is rejected or evicted. +message LiveLimits { + int32 max_sessions = 1; + int32 spans_per_trace = 2; + int32 spans_per_session = 3; + int32 traces_per_session = 4; + int64 max_bytes = 5; + google.protobuf.Duration completion_grace = 6; + google.protobuf.Duration idle_timeout = 7; + google.protobuf.Duration session_ttl = 8; +} + +// Which judge provider API keys the server has in its environment. Values are never returned. +message JudgeKeys { + bool google = 1; + bool anthropic = 2; + bool openai = 3; +} + +message GetConfigResponse { + Ports ports = 1; + LiveLimits live_limits = 2; + // Whether run storage is configured, so RunService works. + bool runs_enabled = 3; + // Whether evaluation results are emitted as gen_ai.evaluation.result events. + bool evaluation_events_enabled = 4; + JudgeKeys judge_keys = 5; +} diff --git a/proto/agentevals/api/v1alpha1/views.proto b/proto/agentevals/api/v1alpha1/views.proto new file mode 100644 index 00000000..56fe7407 --- /dev/null +++ b/proto/agentevals/api/v1alpha1/views.proto @@ -0,0 +1,213 @@ +syntax = "proto3"; + +package agentevals.api.v1alpha1; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; +import "opentelemetry/proto/trace/v1/trace.proto"; + +option go_package = "github.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1"; + +// Views are built by the server from GenAI telemetry, so every client renders the same +// conversation without deriving it. Field comments name the semantic convention attribute a +// field comes from; GenAI attributes are at development stability. +// +// Ids are lowercase hex: 32 characters for trace ids, 16 for span ids. Texts and tool +// arguments or results longer than 64 KiB are cut, with a "truncated" gap. + +// Token counts. Unset means not recorded; 0 means zero. +message Usage { + // gen_ai.usage.input_tokens, which includes cached input tokens. + optional int64 input_tokens = 1; + // gen_ai.usage.output_tokens. + optional int64 output_tokens = 2; + // gen_ai.usage.cache_read.input_tokens. + optional int64 cache_read_input_tokens = 3; + // gen_ai.usage.cache_write.input_tokens. + optional int64 cache_write_input_tokens = 4; + // gen_ai.usage.reasoning.output_tokens. + optional int64 reasoning_output_tokens = 5; +} + +// Why something is missing or approximate. Codes are stable; reasons are for people and never +// carry message content. +message Gap { + string code = 1; + string reason = 2; +} + +// An evaluation result with the fields of the gen_ai.evaluation.result event. +message EvaluationResult { + // gen_ai.evaluation.name. + string name = 1; + // gen_ai.evaluation.score.value. + optional double score_value = 2; + // gen_ai.evaluation.score.label. + optional string score_label = 3; + // gen_ai.evaluation.explanation. + optional string explanation = 4; + // error.type, when the evaluation itself failed. + optional string error_type = 5; + // The judged span. + string trace_id = 6; + string span_id = 7; + // gen_ai.response.id of the judged response. + string response_id = 8; + // service.name of the producer, for results found in telemetry. + string source_service = 9; +} + +message ToolCallView { + // The outermost span of the call. Empty when the call is only known from message parts. + string span_id = 1; + // gen_ai.tool.name. + string name = 2; + // gen_ai.tool.call.id. + optional string call_id = 3; + // gen_ai.tool.call.arguments, parsed. Unset when not recorded. + google.protobuf.Value arguments = 4; + // gen_ai.tool.call.result, parsed, paired with the call by call id and then by order. Unset + // when not recorded. + google.protobuf.Value result = 5; + bool is_error = 6; + // error.type. + string error_type = 7; + google.protobuf.Timestamp started_at = 8; + google.protobuf.Duration duration = 9; +} + +// One logical model call: a leaf inference span with its single leaf wrappers. +message ModelCallView { + string span_id = 1; + // gen_ai.operation.name. + string operation = 2; + // gen_ai.provider.name. + string provider = 3; + // gen_ai.request.model. + string request_model = 4; + // gen_ai.response.model. + string response_model = 5; + // gen_ai.response.id. + string response_id = 6; + Usage usage = 7; + // gen_ai.response.finish_reasons. + repeated string finish_reasons = 8; + // The span status. + opentelemetry.proto.trace.v1.Status.StatusCode status = 9; + // error.type. + string error_type = 10; + google.protobuf.Timestamp started_at = 11; + google.protobuf.Duration duration = 12; + // gen_ai.agent.name of the agent that made the call. + string agent_name = 13; +} + +message TurnView { + // Position in the session or conversation, from 0. + int32 index = 1; + // The turn's anchor: an agent or workflow span, or the trace root. + string trace_id = 2; + string span_id = 3; + // The anchor's span status. + opentelemetry.proto.trace.v1.Status.StatusCode status = 4; + // error.type. + string error_type = 5; + google.protobuf.Timestamp started_at = 6; + google.protobuf.Duration duration = 7; + // gen_ai.agent.name of the anchor. + string agent_name = 8; + // The user message that started the turn (gen_ai.input.messages). Unset when not recorded. + optional string user_text = 9; + // The final answer (gen_ai.output.messages). Unset when not recorded. + optional string answer_text = 10; + // Whether message content was recorded for this turn. + bool content_captured = 11; + repeated ToolCallView tool_calls = 12; + repeated ModelCallView model_calls = 13; + // Sum over model_calls. + Usage usage = 14; + // gen_ai.evaluation.result events other producers attached to this turn's spans. + repeated EvaluationResult external_evaluations = 15; + repeated Gap gaps = 16; +} + +enum SessionState { + SESSION_STATE_UNSPECIFIED = 0; + // Spans may still arrive. + SESSION_STATE_LIVE = 1; + // The completion grace or the idle timeout has passed. New spans reopen the session. + SESSION_STATE_COMPLETE = 2; +} + +enum ContentCapture { + // No turns yet. + CONTENT_CAPTURE_UNSPECIFIED = 0; + CONTENT_CAPTURE_ON = 1; + CONTENT_CAPTURE_OFF = 2; + // Some turns have content, others do not. + CONTENT_CAPTURE_PARTIAL = 3; +} + +message SessionTotals { + int32 turns = 1; + // Turns whose status is ERROR. + int32 failed_turns = 2; + int32 model_calls = 3; + int32 tool_calls = 4; + // Sum over all model calls. + Usage usage = 5; + // Model calls that recorded token usage, for "measured in N of M". + int32 model_calls_with_usage = 6; + // Tool calls that recorded arguments. + int32 tool_calls_with_arguments = 7; +} + +message SessionSummary { + string id = 1; + // The last 6 characters of a UUID or hex id, which are often time ordered so their first + // characters collide; any other id, such as a session name, as it is. + string short_id = 2; + // The resource's gen_ai.agent.name, else the first turn's agent, else the resource's + // service.name. + string agent = 3; + // The resource's gen_ai.agent.version, else the first turn's, else service.version. + string agent_version = 4; + SessionState state = 5; + google.protobuf.Timestamp started_at = 6; + google.protobuf.Timestamp ended_at = 7; + google.protobuf.Duration duration = 8; + // The first turn's user text. Unset when not recorded. + optional string first_user_text = 9; + ContentCapture content_capture = 10; + SessionTotals totals = 11; + // gen_ai.response.model of each call, else gen_ai.request.model, distinct, in first use + // order. + repeated string models = 12; + // The resource of the service that produced the session. + opentelemetry.proto.resource.v1.Resource resource = 13; + // gen_ai.conversation.id. + string conversation_id = 14; + repeated string trace_ids = 15; + // The eval set the session was recorded for, when the producer named one. + string eval_set_id = 16; + int32 span_count = 17; + repeated Gap gaps = 18; +} + +message Session { + SessionSummary summary = 1; + repeated TurnView turns = 2; +} + +// Traces that form one evaluation unit, as conversation turns. +message ConversationView { + // The unit's key: its earliest trace id, or its conversation key. + string key = 1; + // gen_ai.conversation.id. + string conversation_id = 2; + repeated string trace_ids = 3; + repeated TurnView turns = 4; + repeated Gap gaps = 5; +} diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml new file mode 100644 index 00000000..e78142d0 --- /dev/null +++ b/proto/buf.gen.yaml @@ -0,0 +1,25 @@ +version: v2 +# Never set `clean`: the Python output root is ../src. `make proto-generate` removes the +# generated directories instead. +inputs: + - directory: . +plugins: + - remote: buf.build/protocolbuffers/go:v1.36.11 + out: ../gen/go + opt: + - paths=source_relative + - remote: buf.build/grpc/go:v1.6.2 + out: ../gen/go + opt: + - paths=source_relative + - remote: buf.build/protocolbuffers/python:v33.5 + out: ../src + - remote: buf.build/protocolbuffers/pyi:v33.5 + out: ../src + # The stubs call grpcio APIs of this version (registered method handlers); keep it equal to the + # grpcio floor in pyproject.toml. + - remote: buf.build/grpc/python:v1.78.0 + out: ../src + - remote: buf.build/bufbuild/es:v2.13.0 + out: ../ui/src/generated + include_imports: true diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 00000000..9f143543 --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/opentelemetry/opentelemetry + commit: 648a3e2f02e14fe187656ea4ac3befa1 + digest: b5:a0514be587ab2e8598f7102dfdaba5e985138b8c041c707e470a9c8b877410ada6e60cf2359352302f08c0504de685379f7f7a085d4699d69a2e6ed7035e78d9 diff --git a/proto/buf.yaml b/proto/buf.yaml new file mode 100644 index 00000000..6a930551 --- /dev/null +++ b/proto/buf.yaml @@ -0,0 +1,12 @@ +version: v2 +modules: + - path: . +deps: + # OTLP proto release that the installed opentelemetry-proto package is generated from. + - buf.build/opentelemetry/opentelemetry:v1.9.0 +lint: + use: + - STANDARD +breaking: + use: + - FILE diff --git a/pyproject.toml b/pyproject.toml index f10682b5..fe79217b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,13 @@ dependencies = [ "opentelemetry-sdk>=1.38", "opentelemetry-exporter-otlp-proto-http>=1.38", "opentelemetry-exporter-otlp-proto-grpc>=1.38", - "grpcio>=1.66", + # Floors of the generated API code in agentevals.api.v1alpha1 (see proto/buf.gen.yaml). + "grpcio>=1.78", + "protobuf>=6.33.5,<7", + "grpcio-health-checking>=1.78", + "grpcio-reflection>=1.78", + "grpcio-status>=1.78", + "opentelemetry-instrumentation-grpc>=0.59b0", ] [project.optional-dependencies] @@ -107,6 +113,7 @@ target-version = "py311" line-length = 120 fix = true src = ["src", "packages", "tests"] +extend-exclude = ["src/agentevals/api/v1alpha1"] [tool.ruff.lint.isort] known-first-party = ["agentevals"] diff --git a/src/agentevals/api/app.py b/src/agentevals/api/app.py index 1dceaa2a..a61c7f31 100644 --- a/src/agentevals/api/app.py +++ b/src/agentevals/api/app.py @@ -3,16 +3,16 @@ from __future__ import annotations import asyncio -import json import logging import os from contextlib import asynccontextmanager from pathlib import Path from typing import TYPE_CHECKING -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse +from starlette.background import BackgroundTask from agentevals import __version__ @@ -30,9 +30,13 @@ if TYPE_CHECKING: from ..streaming.manager import LiveManager + from .rpc.server import ApiGrpcSettings logger = logging.getLogger(__name__) +# Proxies close idle connections; a comment frame keeps the UI stream open between changes. +SSE_HEARTBEAT_SECONDS = 15 + _TRUE_VALUES = {"true", "1", "yes", "on"} _FALSE_VALUES = {"false", "0", "no", "off"} @@ -126,8 +130,22 @@ async def lifespan(app: FastAPI): app.state.run_worker = worker log_registered_sinks() + api_server = None + api_grpc: ApiGrpcSettings | None = getattr(app.state, "api_grpc", None) + if api_grpc is not None and mgr is not None: + from .rpc.server import start_api_server + from .rpc.services import ApiContext + + context = ApiContext(manager=mgr, run_service=getattr(app.state, "run_service", None), ports=api_grpc.ports) + api_server = await start_api_server(context, api_grpc.host, api_grpc.port) + yield + if api_server is not None: + from .rpc.server import stop_api_server + + mgr.close_watchers() + await stop_api_server(api_server) if worker is not None: await worker.stop() repos = getattr(app.state, "repos", None) @@ -147,9 +165,11 @@ def create_app( enable_streaming: bool = False, static_dir: Path | None = None, otlp_ports: OtlpReceiverPorts | None = None, + api_grpc: ApiGrpcSettings | None = None, ) -> FastAPI: """Create the main agentevals API app. ``otlp_ports`` are the receivers started next to it, - reported by ``/api/config`` so the UI can say where to send telemetry.""" + reported by ``/api/config`` so the UI can say where to send telemetry. With ``api_grpc`` and a + trace manager, the lifespan also serves the gRPC API.""" from ..evaluator.resolver import require_index_membership require_index_membership() @@ -189,6 +209,7 @@ def create_app( if trace_manager is not None: app.state.trace_manager = trace_manager app.state.otlp_ports = otlp_ports + app.state.api_grpc = api_grpc if enable_streaming: if trace_manager is None: @@ -200,28 +221,35 @@ def create_app( @app.get("/stream/ui-updates") async def ui_updates_stream(request: Request): + """Live sessions as protojson ``agentevals.api.v1alpha1.WatchSessionsResponse`` frames, with + turns: every current session, then ``synced``, then each change, as ``WatchSessions``.""" + from ..streaming.manager import TooManyWatchers + mgr = request.app.state.trace_manager + try: + watcher = mgr.subscribe(include_turns=True) + except TooManyWatchers as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc - async def event_generator(): - client = mgr.register_sse_client() + async def frames(): try: while True: - event = await client.queue.get() + try: + event = await asyncio.wait_for(watcher.next(), SSE_HEARTBEAT_SECONDS) + except TimeoutError: + yield ": keepalive\n\n" + continue if event is None: - break - yield f"data: {json.dumps(event)}\n\n" - except asyncio.CancelledError: - pass + return + yield f"data: {event.json}\n\n" finally: - mgr.unregister_sse_client(client) + mgr.unsubscribe(watcher) return StreamingResponse( - event_generator(), + frames(), media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - }, + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + background=BackgroundTask(mgr.unsubscribe, watcher), ) if static_dir is None: diff --git a/src/agentevals/api/evaluation.py b/src/agentevals/api/evaluation.py new file mode 100644 index 00000000..b28fc208 --- /dev/null +++ b/src/agentevals/api/evaluation.py @@ -0,0 +1,193 @@ +"""Trace evaluation as a stream of events, shared by the REST evaluate routes and the gRPC API. + +The evaluation runs in its own task with the request's credentials, so the resolved secrets +never leak into the caller's context and a stream that is closed early cancels the work. The +consumer always receives a terminal :class:`Done` or the evaluation's exception. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from google.adk.evaluation.eval_set import EvalSet + +from ..config import EvalParams +from ..genai.extract import extract_conversation +from ..otel.model import Trace +from ..resolvers import reset_resolved_credentials, resolve_credential_refs, set_resolved_credentials +from ..runner import RunResult, TraceResult, evaluation_groups, run_evaluation_from_traces +from ..trace_metrics import extract_performance_metrics, extract_trace_metadata + +if TYPE_CHECKING: + from ..run.service import RunService + +logger = logging.getLogger(__name__) + + +class CredentialResolutionError(ValueError): + """A credential reference of the request could not be resolved.""" + + +@dataclass +class Progress: + message: str + + +@dataclass +class Performance: + """One evaluation group's performance metrics, before it is evaluated.""" + + trace_id: str + performance_metrics: dict[str, Any] + trace_metadata: dict[str, Any] + + +@dataclass +class GroupResult: + """A group's results so far. A copy: the runner keeps appending to its own object.""" + + result: TraceResult + + +@dataclass +class Done: + result: RunResult + + +EvaluationEvent = Progress | Performance | GroupResult | Done + + +@dataclass +class Evaluation: + traces: list[Trace] + params: EvalParams + eval_set: EvalSet | None = None + credential_refs: dict[str, dict[str, Any]] | None = None + load_errors: list[str] = field(default_factory=list) + # Recorded with the run when storage is configured. + eval_set_dict: dict[str, Any] | None = None + trace_format: str | None = None + upload_filenames: list[str] | None = None + + +def performance_events(traces: list[Trace], group_by: str, eval_set: EvalSet | None) -> list[Performance]: + events = [] + for key, members in evaluation_groups(traces, group_by, eval_set): + try: + conversation = extract_conversation(members, key) + events.append( + Performance( + trace_id=members[0].trace_id, + performance_metrics=extract_performance_metrics(conversation, members), + trace_metadata=extract_trace_metadata(members, conversation), + ) + ) + except Exception: + logger.exception("Failed to extract early performance metrics for %s", key) + return events + + +async def stream_evaluation( + evaluation: Evaluation, *, run_service: RunService | None = None +) -> AsyncIterator[EvaluationEvent]: + """Performance per group, then progress and group results, then :class:`Done`. + + Raises :class:`CredentialResolutionError` after the performance events when a credential + reference cannot be resolved; nothing is evaluated then. + """ + for event in performance_events(evaluation.traces, evaluation.params.group_by, evaluation.eval_set): + yield event + resolved = await _resolve(evaluation.credential_refs) + + queue: asyncio.Queue[EvaluationEvent | BaseException] = asyncio.Queue() + + async def on_progress(message: str) -> None: + await queue.put(Progress(message)) + + async def on_group(result: TraceResult) -> None: + await queue.put(GroupResult(result.model_copy(deep=True))) + + async def run() -> None: + token = set_resolved_credentials(resolved) if resolved else None + try: + if evaluation.traces: + result = await run_evaluation_from_traces( + traces=evaluation.traces, + config=evaluation.params, + eval_set=evaluation.eval_set, + progress_callback=on_progress, + trace_progress_callback=on_group, + ) + else: + result = RunResult(errors=["No traces loaded."]) + if evaluation.load_errors: + result.errors = evaluation.load_errors + result.errors + await queue.put(Done(result)) + except Exception as exc: + await queue.put(exc) + finally: + if token is not None: + reset_resolved_credentials(token) + + task = asyncio.create_task(run()) + try: + while True: + item = await queue.get() + if isinstance(item, BaseException): + raise item + if isinstance(item, Done): + run_id = await persist_evaluation(run_service, evaluation, item.result) + if run_id: + item.result.run_id = run_id + yield item + return + yield item + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +async def _resolve(refs: dict[str, dict[str, Any]] | None) -> dict[str, str] | None: + if not refs: + return None + try: + return await resolve_credential_refs(refs) + except ValueError as exc: + raise CredentialResolutionError(str(exc)) from exc + + +async def evaluate(evaluation: Evaluation, *, run_service: RunService | None = None) -> RunResult: + async for event in stream_evaluation(evaluation, run_service=run_service): + if isinstance(event, Done): + return event.result + raise RuntimeError("evaluation ended without a result") + + +async def persist_evaluation( + run_service: RunService | None, + evaluation: Evaluation, + result: RunResult, +) -> str | None: + """Record a finished evaluation as a run when storage is configured. Best effort: the caller + gets its result even when recording fails.""" + if run_service is None: + return None + try: + run = await run_service.record_eval_run( + params=evaluation.params, + eval_set_dict=evaluation.eval_set_dict, + trace_format=evaluation.trace_format, + upload_filenames=evaluation.upload_filenames, + run_result=result, + ) + return str(run.run_id) + except Exception: + logger.exception("Failed to record the evaluation as a run; the result is still returned") + return None diff --git a/src/agentevals/api/models.py b/src/agentevals/api/models.py index c81df17d..7461b649 100644 --- a/src/agentevals/api/models.py +++ b/src/agentevals/api/models.py @@ -80,7 +80,6 @@ class SessionInfo(CamelModel): is_complete: bool started_at: str metadata: dict[str, Any] = Field(default_factory=dict) - invocations: list[dict[str, Any]] | None = None class CreateEvalSetData(CamelModel): @@ -209,25 +208,3 @@ class SSEDoneEvent(CamelModel): class SSEErrorEvent(CamelModel): error: str - - -# --------------------------------------------------------------------------- -# Live UI broadcast event models -# --------------------------------------------------------------------------- - - -class WSSessionStartedEvent(CamelModel): - type: str = "session_started" - session: SessionInfo - - -class WSSessionCompleteEvent(CamelModel): - type: str = "session_complete" - session_id: str - invocations: list[dict[str, Any]] - - -class WSSessionRemovedEvent(CamelModel): - type: str = "session_removed" - session_id: str - absorbed_by: str | None = None diff --git a/src/agentevals/api/otlp_grpc.py b/src/agentevals/api/otlp_grpc.py index 1713be17..d8e34b4b 100644 --- a/src/agentevals/api/otlp_grpc.py +++ b/src/agentevals/api/otlp_grpc.py @@ -73,6 +73,11 @@ async def Export(self, request, context): # noqa: N802 (gRPC method name) return build_logs_response(result) +def listen_address(host: str, port: int) -> str: + """``host:port`` for grpcio, with an IPv6 host in brackets.""" + return f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + + def create_otlp_grpc_server( host: str, port: int, @@ -98,7 +103,7 @@ def create_otlp_grpc_server( trace_service_pb2_grpc.add_TraceServiceServicer_to_server(OtlpTraceService(manager), server) logs_service_pb2_grpc.add_LogsServiceServicer_to_server(OtlpLogsService(manager), server) - listen_addr = f"{host}:{port}" + listen_addr = listen_address(host, port) bound_port = server.add_insecure_port(listen_addr) if bound_port == 0: raise RuntimeError(f"Failed to bind OTLP gRPC receiver to {listen_addr}") diff --git a/src/agentevals/api/redact.py b/src/agentevals/api/redact.py new file mode 100644 index 00000000..0b970316 --- /dev/null +++ b/src/agentevals/api/redact.py @@ -0,0 +1,77 @@ +"""Secrets out of run specs on every read path. + +A run is stored with what the worker needs: fetch headers, sink credentials, webhook URLs. +Callers get it back without them. Only the transport edge redacts; repositories and the worker +keep the real values. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from ..storage.models import Run, RunSpec + +REDACTED = "REDACTED" + +_SECRET_KEY_PARTS = ("authorization", "password", "secret", "token", "api_key", "apikey", "private_key", "dsn") + + +def redact_run(run: Run) -> Run: + return run.model_copy(update={"spec": redact_spec(run.spec)}) + + +def redact_spec(spec: RunSpec) -> RunSpec: + update: dict[str, Any] = { + "context": _redact(spec.context), + "sinks": [_redact(sink) for sink in spec.sinks], + } + if spec.target.base_url: + update["target"] = spec.target.model_copy(update={"base_url": _without_credentials(spec.target.base_url)}) + return spec.model_copy(update=update) + + +def _redact(value: Any) -> Any: + if isinstance(value, dict): + out: dict[str, Any] = {} + for key, item in value.items(): + name = str(key).lower() + if name == "headers" and isinstance(item, dict): + out[key] = {header: _header_value(v) for header, v in item.items()} + elif any(part in name for part in _SECRET_KEY_PARTS) and not isinstance(item, (dict, list)): + out[key] = REDACTED if item is not None else None + elif name == "url" and isinstance(item, str): + out[key] = _origin(item) + else: + out[key] = _redact(item) + return out + if isinstance(value, list): + return [_redact(item) for item in value] + return value + + +def _header_value(value: Any) -> Any: + """A literal header value is a secret; a ``{"from_env": NAME}`` reference is not.""" + if isinstance(value, dict) and set(value) == {"from_env"}: + return value + return REDACTED + + +def _origin(url: str) -> str: + """Scheme and host only: webhook URLs often carry their token in the path or query.""" + parts = urlsplit(url) + if not parts.scheme or not parts.hostname: + return REDACTED + host = parts.hostname if parts.port is None else f"{parts.hostname}:{parts.port}" + rest = "/" + REDACTED if (parts.path not in ("", "/") or parts.query) else "" + return f"{parts.scheme}://{host}{rest}" + + +def _without_credentials(url: str) -> str: + """The URL without userinfo or query, either of which can carry a key.""" + parts = urlsplit(url) + if parts.username is None and parts.password is None and not parts.query: + return url + host = parts.hostname or "" + netloc = host if parts.port is None else f"{host}:{parts.port}" + return urlunsplit((parts.scheme, netloc, parts.path, "", "")) diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 17e670f2..2eee1214 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -2,13 +2,14 @@ from __future__ import annotations -import asyncio +import dataclasses import json import logging import os import re import shutil import tempfile +from collections.abc import AsyncIterator from contextlib import contextmanager from typing import Any @@ -19,7 +20,7 @@ from agentevals import __version__ from ..adk_bridge import invocation_to_dict, to_adk_invocations -from ..builtin_metrics import METRICS_NEEDING_EXPECTED, METRICS_NEEDING_GCP, METRICS_NEEDING_LLM +from ..builtin_metrics import metric_catalog from ..config import EvalParams, EvalRunConfig from ..genai.extract import extract_conversation from ..loader import load_traces @@ -29,15 +30,26 @@ resolve_credential_refs, set_resolved_credentials, ) +from ..run.service import RunService from ..runner import ( RunResult, evaluation_groups, load_eval_set, load_eval_set_from_dict, run_evaluation, - run_evaluation_from_traces, ) -from ..trace_metrics import extract_performance_metrics, extract_trace_metadata +from ..trace_metrics import extract_trace_metadata +from .evaluation import ( + CredentialResolutionError, + Evaluation, + EvaluationEvent, + GroupResult, + Performance, + Progress, + evaluate, + persist_evaluation, + stream_evaluation, +) from .models import ( ApiKeyStatus, ConfigData, @@ -127,32 +139,6 @@ def _camel_keys(obj: Any) -> Any: return obj -def _group_views(key: str, traces: list) -> tuple[dict[str, Any], dict[str, Any]]: - """Performance metrics and metadata of one evaluation group, as camelCase wire dicts.""" - conversation = extract_conversation(traces, key) - return ( - _camel_keys(extract_performance_metrics(conversation, traces)), - _camel_keys(extract_trace_metadata(traces, conversation)), - ) - - -def _performance_events(traces: list, group_by: str, eval_set) -> list[str]: - """One early ``performance_metrics`` event per evaluation group, keyed by the group's trace id.""" - events = [] - for key, members in evaluation_groups(traces, group_by, eval_set): - try: - perf_metrics, trace_metadata = _group_views(key, members) - evt = SSEPerformanceMetricsEvent( - trace_id=members[0].trace_id, - performance_metrics=perf_metrics, - trace_metadata=trace_metadata, - ) - events.append(f"event: performance_metrics\ndata: {evt.model_dump_json(by_alias=True)}\n\n") - except Exception as e: - logger.error(f"Failed to extract early performance metrics for {key}: {e}") - return events - - def _safe_upload_path(temp_dir: str, filename: str, idx: int = 0) -> str: """Build a save path inside ``temp_dir`` from a client-controlled filename. @@ -179,40 +165,129 @@ def _load_eval_set_dict(path: str | None) -> dict | None: return None -async def _maybe_persist_evaluate_run( - request: Request, - *, - params: "EvalParams", - eval_set_dict: dict | None, - trace_format: str | None, - upload_filenames: list[str] | None, - run_result: "RunResult", -) -> str | None: - """Best-effort: persist the just-completed eval as a Run row + Result - rows when ``app.state.run_service`` is configured (postgres backend). - Returns the synthesized ``run_id`` for inclusion in the response, or - ``None`` on the memory backend or on persistence failure (eval result - is still returned to the caller in that case).""" - service = getattr(request.app.state, "run_service", None) - if service is None: +def _run_service(request: Request) -> RunService | None: + return getattr(request.app.state, "run_service", None) + + +def _config_json(config: str) -> dict[str, Any]: + try: + return json.loads(config) + except json.JSONDecodeError as exc: + raise HTTPException(status_code=400, detail=f"Invalid config JSON: {exc}") from exc + + +def _credential_refs_form(raw: str | None) -> dict[str, dict[str, Any]] | None: + try: + return _parse_credential_refs_form(raw) + except ValueError as exc: + raise HTTPException(status_code=400, detail=f"Invalid credentialRefs: {exc}") from exc + + +async def _save_upload(upload: UploadFile, path: str, what: str) -> None: + content = await upload.read() + if len(content) > _MAX_UPLOAD_BYTES: + raise HTTPException(status_code=400, detail=f"{what} exceeds 10MB limit") + with open(path, "wb") as f: # noqa: ASYNC230 + f.write(content) + + +async def _save_trace_uploads(trace_files: list[UploadFile], temp_dir: str) -> list[str]: + paths = [] + for idx, trace_file in enumerate(trace_files): + if not trace_file.filename: + continue + if not trace_file.filename.lower().endswith((".json", ".jsonl")): + raise HTTPException( + status_code=400, + detail=f"Invalid file extension for {trace_file.filename}. Only .json and .jsonl files are allowed.", + ) + path = _safe_upload_path(temp_dir, trace_file.filename, idx) + await _save_upload(trace_file, path, f"File {trace_file.filename}") + paths.append(path) + if not paths: + raise HTTPException(status_code=400, detail="No valid trace files provided") + return paths + + +async def _save_eval_set_upload(eval_set_file: UploadFile | None, temp_dir: str) -> str | None: + if not eval_set_file or not eval_set_file.filename: return None + if not eval_set_file.filename.endswith(".json"): + raise HTTPException( + status_code=400, detail="Invalid file extension for eval set. Only .json files are allowed." + ) + path = os.path.join(temp_dir, f"evalset_{os.path.basename(eval_set_file.filename)}") + await _save_upload(eval_set_file, path, "Eval set file") + return path + + +def _eval_run_config(config_dict: dict[str, Any], trace_paths: list[str], eval_set_path: str | None) -> EvalRunConfig: try: - run = await service.record_eval_run( - params=params, - eval_set_dict=eval_set_dict, - trace_format=trace_format, - upload_filenames=upload_filenames, - run_result=run_result, + return EvalRunConfig.model_validate( + { + **config_dict, + "traceFiles": trace_paths, + "evalSetFile": eval_set_path, + "traceFormat": config_dict.get("trace_format"), + } ) - return str(run.run_id) - except Exception: - logger.exception("failed to persist /api/evaluate run; eval result still returned to caller") - return None + except Exception as exc: + raise HTTPException(status_code=400, detail=f"Invalid config: {exc}") from exc + + +def _sse(event: Any) -> str: + return f"data: {event.model_dump_json(by_alias=True)}\n\n" + + +def _sse_error(message: str) -> str: + return _sse(SSEErrorEvent(error=message)) + + +def _sse_event(event: EvaluationEvent) -> str: + if isinstance(event, Performance): + payload = SSEPerformanceMetricsEvent( + trace_id=event.trace_id, + performance_metrics=_camel_keys(event.performance_metrics), + trace_metadata=_camel_keys(event.trace_metadata), + ) + return f"event: performance_metrics\n{_sse(payload)}" + if isinstance(event, Progress): + return _sse(SSEProgressEvent(message=event.message)) + if isinstance(event, GroupResult): + return _sse( + SSETraceProgressEvent( + trace_progress=SSETraceProgress( + trace_id=event.result.trace_id, + partial_result=_camel_keys(event.result.model_dump(by_alias=True)), + ) + ) + ) + return _sse(SSEDoneEvent(result=_camel_keys(event.result.model_dump(by_alias=True)))) + + +async def _sse_evaluation(evaluation: Evaluation, request: Request) -> AsyncIterator[str]: + try: + async for event in stream_evaluation(evaluation, run_service=_run_service(request)): + yield _sse_event(event) + except CredentialResolutionError as exc: + yield _sse_error(f"Could not resolve credentialRefs: {exc}") + except Exception as exc: + logger.exception("Evaluation stream failed") + yield _sse_error(str(exc)) + + +def _event_stream(events: AsyncIterator[str]) -> StreamingResponse: + return StreamingResponse( + events, + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) router = APIRouter() _MAX_JSON_BODY_BYTES = 50 * 1024 * 1024 # 50 MB (multipart endpoints allow 10 MB per file) +_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 @router.get("/health", response_model=StandardResponse[HealthData]) @@ -236,168 +311,7 @@ async def get_config(request: Request): @router.get("/metrics", response_model=StandardResponse[list[MetricInfo]]) async def list_metrics(): - from ..config import UNSUPPORTED_BUILTIN_METRICS - - def _needs_rubrics(name: str) -> bool: - return name.startswith("rubric_based_") - - _METRIC_CATEGORIES = { - "tool_trajectory_avg_score": "trajectory", - "response_match_score": "response", - "response_evaluation_score": "response", - "final_response_match_v2": "response", - "rubric_based_final_response_quality_v1": "quality", - "rubric_based_tool_use_quality_v1": "quality", - "hallucinations_v1": "safety", - "safety_v1": "safety", - "per_turn_user_simulator_quality_v1": "simulation", - "multi_turn_task_success_v1": "multi-turn", - "multi_turn_trajectory_quality_v1": "multi-turn", - "multi_turn_tool_use_quality_v1": "multi-turn", - "rubric_based_multi_turn_trajectory_quality_v1": "multi-turn", - } - - try: - from google.adk.evaluation.metric_evaluator_registry import ( - DEFAULT_METRIC_EVALUATOR_REGISTRY, - ) - - registry_metrics = DEFAULT_METRIC_EVALUATOR_REGISTRY.get_registered_metrics() - - metrics = [] - for m in registry_metrics: - if m.metric_name in UNSUPPORTED_BUILTIN_METRICS: - continue - - metrics.append( - MetricInfo( - name=m.metric_name, - category=_METRIC_CATEGORIES.get(m.metric_name, "other"), - requires_eval_set=m.metric_name in METRICS_NEEDING_EXPECTED, - requires_llm=m.metric_name in METRICS_NEEDING_LLM, - requires_gcp=m.metric_name in METRICS_NEEDING_GCP, - requires_rubrics=_needs_rubrics(m.metric_name), - description=m.description or "No description available", - working=not _needs_rubrics(m.metric_name), - ) - ) - - return StandardResponse(data=metrics) - - except ImportError: - fallback = [ - MetricInfo( - name="tool_trajectory_avg_score", - category="trajectory", - requires_eval_set=True, - requires_llm=False, - requires_gcp=False, - requires_rubrics=False, - working=True, - description="Compare tool call sequences against expected trajectory", - ), - MetricInfo( - name="response_match_score", - category="response", - requires_eval_set=True, - requires_llm=False, - requires_gcp=False, - requires_rubrics=False, - working=True, - description="Text similarity between actual and expected responses using ROUGE-1", - ), - MetricInfo( - name="response_evaluation_score", - category="response", - requires_eval_set=True, - requires_llm=False, - requires_gcp=True, - requires_rubrics=False, - working=True, - description="Semantic evaluation of response quality using Vertex AI", - ), - MetricInfo( - name="final_response_match_v2", - category="response", - requires_eval_set=True, - requires_llm=True, - requires_gcp=False, - requires_rubrics=False, - working=True, - description="LLM-based comparison of final responses", - ), - MetricInfo( - name="hallucinations_v1", - category="safety", - requires_eval_set=False, - requires_llm=True, - requires_gcp=False, - requires_rubrics=False, - working=True, - description="Detect hallucinated information in responses", - ), - MetricInfo( - name="safety_v1", - category="safety", - requires_eval_set=False, - requires_llm=False, - requires_gcp=True, - requires_rubrics=False, - working=True, - description="Safety and security assessment using Vertex AI", - ), - MetricInfo( - name="rubric_based_final_response_quality_v1", - category="quality", - requires_eval_set=False, - requires_llm=True, - requires_gcp=False, - requires_rubrics=True, - working=False, - description="Rubric-based quality assessment of responses (requires rubrics config)", - ), - MetricInfo( - name="rubric_based_tool_use_quality_v1", - category="quality", - requires_eval_set=False, - requires_llm=True, - requires_gcp=False, - requires_rubrics=True, - working=False, - description="Rubric-based assessment of tool usage quality (requires rubrics config)", - ), - MetricInfo( - name="multi_turn_task_success_v1", - category="multi-turn", - requires_eval_set=False, - requires_llm=False, - requires_gcp=True, - requires_rubrics=False, - working=True, - description="Evaluates if the agent achieved the goal(s) of the multi-turn conversation (Vertex AI)", - ), - MetricInfo( - name="multi_turn_trajectory_quality_v1", - category="multi-turn", - requires_eval_set=False, - requires_llm=False, - requires_gcp=True, - requires_rubrics=False, - working=True, - description="Evaluates the overall trajectory the agent took across the conversation (Vertex AI)", - ), - MetricInfo( - name="multi_turn_tool_use_quality_v1", - category="multi-turn", - requires_eval_set=False, - requires_llm=False, - requires_gcp=True, - requires_rubrics=False, - working=True, - description="Evaluates function calls made during a multi-turn conversation (Vertex AI)", - ), - ] - return StandardResponse(data=fallback) + return StandardResponse(data=[MetricInfo(**dataclasses.asdict(m)) for m in metric_catalog()]) @router.post("/validate/eval-set", response_model=StandardResponse[EvalSetValidation]) @@ -516,7 +430,9 @@ async def convert_trace_files( first = members[0] conversation = extract_conversation(members, key) invocations = [_serialize_invocation(inv) for inv in to_adk_invocations(conversation.turns)] - warnings = [w for t in members for w in t.warnings] + [w for t in conversation.turns for w in t.warnings] + warnings = [w for t in members for w in t.warnings] + [ + w.reason for t in conversation.turns for w in t.warnings + ] meta_dict = extract_trace_metadata(members, conversation) meta = TraceConversionMetadata( agent_name=meta_dict.get("agent_name"), @@ -556,92 +472,18 @@ async def evaluate_traces( eval_set_file: UploadFile | None = File(None), credential_refs: str | None = Form(None), ): - """ - Evaluate agent traces using the provided evaluator configuration. + """Evaluate trace files with the given evaluator configuration (a JSON string). - Args: - trace_files: List of Jaeger or OTLP JSON trace files - config: JSON string with evaluation configuration - eval_set_file: Optional golden eval set file - credential_refs: Optional JSON string mapping logical credential names to - secret references, resolved so LLM-as-Judge graders can authenticate - - Returns: - RunResult with trace results and any errors + ``credential_refs`` is an optional JSON object mapping logical credential names to secret + references, resolved so LLM as judge graders can authenticate. """ temp_dir = tempfile.mkdtemp() try: - try: - config_dict = json.loads(config) - except json.JSONDecodeError as exc: - raise HTTPException(status_code=400, detail=f"Invalid config JSON: {exc}") from exc - - try: - cred_refs = _parse_credential_refs_form(credential_refs) - except ValueError as exc: - raise HTTPException(status_code=400, detail=f"Invalid credentialRefs: {exc}") from exc - - trace_paths = [] - for idx, trace_file in enumerate(trace_files): - if not trace_file.filename: - continue - - if not (trace_file.filename.endswith(".json") or trace_file.filename.endswith(".jsonl")): - raise HTTPException( - status_code=400, - detail=f"Invalid file extension for {trace_file.filename}. Only .json and .jsonl files are allowed.", - ) - - trace_path = _safe_upload_path(temp_dir, trace_file.filename, idx) - with open(trace_path, "wb") as f: # noqa: ASYNC230 - content = await trace_file.read() - - if len(content) > 10 * 1024 * 1024: - raise HTTPException( - status_code=400, - detail=f"File {trace_file.filename} exceeds 10MB limit", - ) - - f.write(content) - trace_paths.append(trace_path) - - if not trace_paths: - raise HTTPException( - status_code=400, - detail="No valid trace files provided", - ) - - trace_format = config_dict.get("trace_format") - - eval_set_path = None - if eval_set_file and eval_set_file.filename: - if not eval_set_file.filename.endswith(".json"): - raise HTTPException( - status_code=400, - detail="Invalid file extension for eval set. Only .json files are allowed.", - ) - - eval_set_path = os.path.join(temp_dir, f"evalset_{os.path.basename(eval_set_file.filename)}") - with open(eval_set_path, "wb") as f: # noqa: ASYNC230 - content = await eval_set_file.read() - if len(content) > 10 * 1024 * 1024: - raise HTTPException( - status_code=400, - detail="Eval set file exceeds 10MB limit", - ) - f.write(content) - - try: - eval_config = EvalRunConfig.model_validate( - { - **config_dict, - "traceFiles": trace_paths, - "evalSetFile": eval_set_path, - "traceFormat": trace_format, - } - ) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Invalid config: {exc}") from exc + config_dict = _config_json(config) + cred_refs = _credential_refs_form(credential_refs) + trace_paths = await _save_trace_uploads(trace_files, temp_dir) + eval_set_path = await _save_eval_set_upload(eval_set_file, temp_dir) + eval_config = _eval_run_config(config_dict, trace_paths, eval_set_path) logger.info( "Evaluating %d trace file(s) with evaluators: %s", @@ -652,26 +494,26 @@ async def evaluate_traces( with _scoped_credentials(resolved_creds): result = await run_evaluation(eval_config) - run_id = await _maybe_persist_evaluate_run( - request, - params=eval_config, - eval_set_dict=_load_eval_set_dict(eval_set_path), - trace_format=eval_config.trace_format, - upload_filenames=[tf.filename for tf in trace_files if tf.filename], - run_result=result, + run_id = await persist_evaluation( + _run_service(request), + Evaluation( + traces=[], + params=eval_config, + eval_set_dict=_load_eval_set_dict(eval_set_path), + trace_format=eval_config.trace_format, + upload_filenames=[tf.filename for tf in trace_files if tf.filename], + ), + result, ) if run_id: result.run_id = run_id - - result_dict = _camel_keys(result.model_dump(by_alias=True)) - return StandardResponse(data=result_dict) + return StandardResponse(data=_camel_keys(result.model_dump(by_alias=True))) except HTTPException: raise except Exception as exc: logger.exception("Evaluation failed") raise HTTPException(status_code=500, detail=f"Internal error: {exc!s}") from exc - finally: shutil.rmtree(temp_dir) @@ -688,178 +530,48 @@ async def evaluate_traces_stream( temp_dir = tempfile.mkdtemp() upload_filenames = [tf.filename for tf in trace_files if tf.filename] - async def event_generator(): + async def events(): try: try: - config_dict = json.loads(config) - except json.JSONDecodeError as exc: - yield f"data: {SSEErrorEvent(error=f'Invalid config JSON: {exc}').model_dump_json(by_alias=True)}\n\n" - return - - try: - cred_refs = _parse_credential_refs_form(credential_refs) - except ValueError as exc: - yield f"data: {SSEErrorEvent(error=f'Invalid credentialRefs: {exc}').model_dump_json(by_alias=True)}\n\n" - return - - trace_paths = [] - for idx, trace_file in enumerate(trace_files): - if not trace_file.filename: - continue - - if not (trace_file.filename.endswith(".json") or trace_file.filename.endswith(".jsonl")): - yield f"data: {SSEErrorEvent(error=f'Invalid file extension for {trace_file.filename}').model_dump_json(by_alias=True)}\n\n" - return - - trace_path = _safe_upload_path(temp_dir, trace_file.filename, idx) - with open(trace_path, "wb") as f: # noqa: ASYNC230 - content = await trace_file.read() - - if len(content) > 10 * 1024 * 1024: - yield f"data: {SSEErrorEvent(error=f'File {trace_file.filename} exceeds 10MB').model_dump_json(by_alias=True)}\n\n" - return - - f.write(content) - trace_paths.append(trace_path) - - if not trace_paths: - yield f"data: {SSEErrorEvent(error='No valid trace files provided').model_dump_json(by_alias=True)}\n\n" - return - - trace_format = config_dict.get("trace_format") - - eval_set_path = None - if eval_set_file and eval_set_file.filename: - if not eval_set_file.filename.endswith(".json"): - yield f"data: {SSEErrorEvent(error='Invalid file extension for eval set').model_dump_json(by_alias=True)}\n\n" - return - - eval_set_path = os.path.join(temp_dir, f"evalset_{os.path.basename(eval_set_file.filename)}") - with open(eval_set_path, "wb") as f: # noqa: ASYNC230 - content = await eval_set_file.read() - if len(content) > 10 * 1024 * 1024: - yield f"data: {SSEErrorEvent(error='Eval set file exceeds 10MB').model_dump_json(by_alias=True)}\n\n" - return - f.write(content) - - try: - eval_config = EvalRunConfig.model_validate( - { - **config_dict, - "traceFiles": trace_paths, - "evalSetFile": eval_set_path, - "traceFormat": trace_format, - } - ) - except Exception as exc: - yield f"data: {SSEErrorEvent(error=f'Invalid config: {exc}').model_dump_json(by_alias=True)}\n\n" + config_dict = _config_json(config) + cred_refs = _credential_refs_form(credential_refs) + trace_paths = await _save_trace_uploads(trace_files, temp_dir) + eval_set_path = await _save_eval_set_upload(eval_set_file, temp_dir) + eval_config = _eval_run_config(config_dict, trace_paths, eval_set_path) + except HTTPException as exc: + yield _sse_error(exc.detail) return - loaded_traces = [] + traces = [] load_errors: list[str] = [] - for trace_file_path in trace_paths: + for path in trace_paths: try: - loaded_traces.extend(load_traces(trace_file_path, format=eval_config.trace_format)) + traces.extend(load_traces(path, format=eval_config.trace_format)) except Exception as exc: - load_errors.append(f"Failed to load trace file '{os.path.basename(trace_file_path)}': {exc}") - stream_eval_set = None + load_errors.append(f"Failed to load trace file '{os.path.basename(path)}': {exc}") + eval_set = None if eval_set_path: try: - stream_eval_set = load_eval_set(eval_set_path) + eval_set = load_eval_set(eval_set_path) except Exception as exc: load_errors.append(f"Failed to load eval set: {exc}") - for event in _performance_events(loaded_traces, eval_config.group_by, stream_eval_set): - yield event - - queue: asyncio.Queue = asyncio.Queue() - - async def progress_callback(message: str): - await queue.put(("progress", message)) - - async def trace_progress_callback(trace_result): - await queue.put(("trace_progress", trace_result)) - - async def run_with_progress(): - if loaded_traces: - result = await run_evaluation_from_traces( - traces=loaded_traces, - config=eval_config, - eval_set=stream_eval_set, - progress_callback=progress_callback, - trace_progress_callback=trace_progress_callback, - ) - else: - result = RunResult(errors=["No traces loaded."]) - if load_errors: - result.errors = load_errors + result.errors - await queue.put(("done", result)) - - try: - resolved_creds = await resolve_credential_refs(cred_refs) if cred_refs else None - except ValueError as exc: - yield f"data: {SSEErrorEvent(error=f'Could not resolve credentialRefs: {exc}').model_dump_json(by_alias=True)}\n\n" - return - - with _scoped_credentials(resolved_creds): - eval_task = asyncio.create_task(run_with_progress()) - - try: - while True: - msg = await queue.get() - tag, payload = msg - - if tag == "done": - run_id = await _maybe_persist_evaluate_run( - request, - params=eval_config, - eval_set_dict=_load_eval_set_dict(eval_set_path), - trace_format=eval_config.trace_format, - upload_filenames=upload_filenames, - run_result=payload, - ) - if run_id: - payload.run_id = run_id - evt = SSEDoneEvent( - result=_camel_keys(payload.model_dump(by_alias=True)), - ) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - break - elif tag == "trace_progress": - evt = SSETraceProgressEvent( - trace_progress=SSETraceProgress( - trace_id=payload.trace_id, - partial_result=_camel_keys(payload.model_dump(by_alias=True)), - ) - ) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - elif tag == "progress": - evt = SSEProgressEvent(message=payload) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - finally: - if not eval_task.done(): - eval_task.cancel() - try: - await eval_task - except asyncio.CancelledError: - pass - - except Exception as exc: - logger.exception("Evaluation stream failed") - evt = SSEErrorEvent(error=str(exc)) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - + evaluation = Evaluation( + traces=traces, + params=eval_config, + eval_set=eval_set, + credential_refs=cred_refs, + load_errors=load_errors, + eval_set_dict=_load_eval_set_dict(eval_set_path), + trace_format=eval_config.trace_format, + upload_filenames=upload_filenames, + ) + async for frame in _sse_evaluation(evaluation, request): + yield frame finally: shutil.rmtree(temp_dir) - return StreamingResponse( - event_generator(), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - }, - ) + return _event_stream(events()) def _parse_json_request(request: EvaluateJsonRequest): @@ -894,8 +606,14 @@ def _check_json_body_size(raw_request: Request): ) -def _sse_error(message: str) -> str: - return f"data: {SSEErrorEvent(error=message).model_dump_json(by_alias=True)}\n\n" +def _json_evaluation(request: EvaluateJsonRequest, traces: list, eval_set) -> Evaluation: + return Evaluation( + traces=traces, + params=request.config, + eval_set=eval_set, + credential_refs=request.credential_refs, + eval_set_dict=request.eval_set, + ) @router.post("/evaluate/json", response_model=StandardResponse[RunResult]) @@ -903,28 +621,11 @@ async def evaluate_traces_json(request: EvaluateJsonRequest, raw_request: Reques """Evaluate OTLP JSON traces passed in the request body.""" _check_json_body_size(raw_request) traces, eval_set = _parse_json_request(request) - resolved_creds = await _resolve_credentials(request.credential_refs) - try: - with _scoped_credentials(resolved_creds): - result = await run_evaluation_from_traces( - traces=traces, - config=request.config, - eval_set=eval_set, - ) - run_id = await _maybe_persist_evaluate_run( - raw_request, - params=request.config, - eval_set_dict=request.eval_set, - trace_format=None, - upload_filenames=None, - run_result=result, - ) - if run_id: - result.run_id = run_id + result = await evaluate(_json_evaluation(request, traces, eval_set), run_service=_run_service(raw_request)) return StandardResponse(data=_camel_keys(result.model_dump(by_alias=True))) - except HTTPException: - raise + except CredentialResolutionError as exc: + raise HTTPException(status_code=400, detail=f"Could not resolve credentialRefs: {exc}") from exc except Exception as exc: logger.exception("JSON evaluation failed") raise HTTPException(status_code=500, detail=f"Internal error: {exc!s}") from exc @@ -935,95 +636,13 @@ async def evaluate_traces_json_stream(request: EvaluateJsonRequest, raw_request: """Evaluate OTLP JSON traces with real-time progress via SSE.""" _check_json_body_size(raw_request) - async def event_generator(): + async def events(): try: - try: - traces, eval_set = _parse_json_request(request) - except HTTPException as exc: - yield _sse_error(exc.detail) - return - - for event in _performance_events(traces, request.config.group_by, eval_set): - yield event - - queue: asyncio.Queue = asyncio.Queue() - - async def progress_callback(message: str): - await queue.put(("progress", message)) - - async def trace_progress_callback(trace_result): - await queue.put(("trace_progress", trace_result)) - - async def run_with_progress(): - result = await run_evaluation_from_traces( - traces=traces, - config=request.config, - eval_set=eval_set, - progress_callback=progress_callback, - trace_progress_callback=trace_progress_callback, - ) - await queue.put(("done", result)) - - try: - resolved_creds = ( - await resolve_credential_refs(request.credential_refs) if request.credential_refs else None - ) - except ValueError as exc: - yield _sse_error(f"Could not resolve credentialRefs: {exc}") - return - - with _scoped_credentials(resolved_creds): - eval_task = asyncio.create_task(run_with_progress()) - - try: - while True: - msg = await queue.get() - tag, payload = msg - - if tag == "done": - run_id = await _maybe_persist_evaluate_run( - raw_request, - params=request.config, - eval_set_dict=request.eval_set, - trace_format=None, - upload_filenames=None, - run_result=payload, - ) - if run_id: - payload.run_id = run_id - evt = SSEDoneEvent( - result=_camel_keys(payload.model_dump(by_alias=True)), - ) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - break - elif tag == "trace_progress": - evt = SSETraceProgressEvent( - trace_progress=SSETraceProgress( - trace_id=payload.trace_id, - partial_result=_camel_keys(payload.model_dump(by_alias=True)), - ) - ) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - elif tag == "progress": - evt = SSEProgressEvent(message=payload) - yield f"data: {evt.model_dump_json(by_alias=True)}\n\n" - finally: - if not eval_task.done(): - eval_task.cancel() - try: - await eval_task - except asyncio.CancelledError: - pass - - except Exception as exc: - logger.exception("JSON evaluation stream failed") - yield _sse_error(str(exc)) - - return StreamingResponse( - event_generator(), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - }, - ) + traces, eval_set = _parse_json_request(request) + except HTTPException as exc: + yield _sse_error(exc.detail) + return + async for frame in _sse_evaluation(_json_evaluation(request, traces, eval_set), raw_request): + yield frame + + return _event_stream(events()) diff --git a/src/agentevals/api/rpc/__init__.py b/src/agentevals/api/rpc/__init__.py new file mode 100644 index 00000000..36048202 --- /dev/null +++ b/src/agentevals/api/rpc/__init__.py @@ -0,0 +1 @@ +"""The API's gRPC services, served next to the REST and UI port.""" diff --git a/src/agentevals/api/rpc/convert.py b/src/agentevals/api/rpc/convert.py new file mode 100644 index 00000000..e62a1102 --- /dev/null +++ b/src/agentevals/api/rpc/convert.py @@ -0,0 +1,428 @@ +"""Conversions between the v1alpha1 messages and the Pydantic models. + +Pydantic stays the only validator: a request becomes a dict with the model's field names and goes +through ``model_validate``. Enum spellings differ per field (``queued``, ``PASSED``, +``otlp-json``), so each enum has an explicit map. ``json_format`` is used for ``Struct`` and +``Value`` payloads only. +""" + +from __future__ import annotations + +import itertools +import json +import logging +import math +from collections.abc import Mapping +from datetime import UTC, datetime +from typing import Any + +from google.protobuf import json_format, struct_pb2, timestamp_pb2 +from pydantic import ValidationError + +from ...config import BuiltinMetricDef, CodeEvaluatorDef, EvalParams, RemoteEvaluatorDef +from ...loader import load_telemetry_text +from ...otel.decode import decode_logs_proto, decode_traces_proto +from ...otel.encode import encode_traces +from ...otel.model import Trace, build_traces +from ...runner import MetricResult, RunResult, TraceResult +from ...storage.models import Result, ResultStatus, Run, RunSpec, RunStatus +from ..redact import redact_run +from ..v1alpha1 import common_pb2, evaluation_pb2, runs_pb2 +from ..views import json_safe +from .policy import ApiError + +logger = logging.getLogger(__name__) + +RUN_STATUS = { + RunStatus.QUEUED: runs_pb2.RUN_STATUS_QUEUED, + RunStatus.RUNNING: runs_pb2.RUN_STATUS_RUNNING, + RunStatus.SUCCEEDED: runs_pb2.RUN_STATUS_SUCCEEDED, + RunStatus.FAILED: runs_pb2.RUN_STATUS_FAILED, + RunStatus.CANCELLED: runs_pb2.RUN_STATUS_CANCELLED, +} +RUN_STATUS_FROM_PB = {v: k for k, v in RUN_STATUS.items()} +RESULT_STATUS = { + ResultStatus.PASSED: runs_pb2.RESULT_STATUS_PASSED, + ResultStatus.FAILED: runs_pb2.RESULT_STATUS_FAILED, + ResultStatus.ERRORED: runs_pb2.RESULT_STATUS_ERRORED, + ResultStatus.SKIPPED: runs_pb2.RESULT_STATUS_SKIPPED, +} +EVALUATOR_TYPE = { + "builtin": runs_pb2.EVALUATOR_TYPE_BUILTIN, + "code": runs_pb2.EVALUATOR_TYPE_CODE, + "remote": runs_pb2.EVALUATOR_TYPE_REMOTE, +} +EVAL_STATUS = { + "PASSED": evaluation_pb2.EVAL_STATUS_PASSED, + "FAILED": evaluation_pb2.EVAL_STATUS_FAILED, + "NOT_EVALUATED": evaluation_pb2.EVAL_STATUS_NOT_EVALUATED, +} +GROUP_BY = { + "auto": common_pb2.GROUP_BY_AUTO, + "trace": common_pb2.GROUP_BY_TRACE, + "conversation": common_pb2.GROUP_BY_CONVERSATION, +} +GROUP_BY_FROM_PB = {v: k for k, v in GROUP_BY.items()} +TRAJECTORY_MATCH = { + "EXACT": evaluation_pb2.TRAJECTORY_MATCH_TYPE_EXACT, + "IN_ORDER": evaluation_pb2.TRAJECTORY_MATCH_TYPE_IN_ORDER, + "ANY_ORDER": evaluation_pb2.TRAJECTORY_MATCH_TYPE_ANY_ORDER, +} +TRAJECTORY_MATCH_FROM_PB = {v: k for k, v in TRAJECTORY_MATCH.items()} +# A JSON lines document is OTLP/JSON for the loaders, which split the lines themselves. +TRACE_FORMAT_FROM_PB = { + common_pb2.TRACE_FORMAT_UNSPECIFIED: None, + common_pb2.TRACE_FORMAT_OTLP_JSON: "otlp-json", + common_pb2.TRACE_FORMAT_OTLP_JSONL: "otlp-json", + common_pb2.TRACE_FORMAT_JAEGER_JSON: "jaeger-json", +} +TRACE_FORMAT = {"otlp-json": common_pb2.TRACE_FORMAT_OTLP_JSON, "jaeger-json": common_pb2.TRACE_FORMAT_JAEGER_JSON} + +MAX_DOCUMENT_BYTES = 10 * 1024 * 1024 + + +# ---------------------------------------------------------------- Struct and Value + + +def to_struct(data: Mapping[str, Any]) -> struct_pb2.Struct: + struct = struct_pb2.Struct() + json_format.ParseDict(json_safe(dict(data)), struct) + return struct + + +def from_struct(struct: struct_pb2.Struct) -> dict[str, Any]: + """The struct as a dict, with whole numbers as ints: a Struct stores every number as a double.""" + return _ints(json_format.MessageToDict(struct)) + + +def _ints(value: Any) -> Any: + if isinstance(value, float) and value.is_integer() and abs(value) <= 2**53: + return int(value) + if isinstance(value, dict): + return {k: _ints(v) for k, v in value.items()} + if isinstance(value, list): + return [_ints(v) for v in value] + return value + + +def _timestamp(value: datetime | None) -> timestamp_pb2.Timestamp | None: + if value is None: + return None + ts = timestamp_pb2.Timestamp() + ts.FromDatetime(value if value.tzinfo else value.replace(tzinfo=UTC)) + return ts + + +def _finite(value: float | None) -> float | None: + return value if value is not None and math.isfinite(value) else None + + +# ---------------------------------------------------------------- traces + + +def load_trace_input(trace_input: common_pb2.TraceInput) -> tuple[list[Trace], list[str]]: + """Traces of the OTLP protobuf and every document, plus a warning per document not read.""" + traces: list[Trace] = [] + warnings: list[str] = [] + if trace_input.HasField("otlp"): + traces.extend(_otlp_traces(trace_input.otlp)) + for index, document in enumerate(trace_input.documents): + name = document.name or f"document {index + 1}" + if len(document.content) > MAX_DOCUMENT_BYTES: + raise ApiError.invalid(f"{name} exceeds 10MB limit") + try: + text = document.content.decode("utf-8") + traces.extend(load_telemetry_text(text, format=TRACE_FORMAT_FROM_PB[document.format], name=name).traces) + except (UnicodeDecodeError, ValueError) as exc: + warnings.append(f"Failed to load {name}: {exc}") + return traces, warnings + + +def _otlp_traces(otlp: common_pb2.OtlpTelemetry) -> list[Trace]: + decoded = decode_traces_proto(otlp.traces) + logs = decode_logs_proto(otlp.logs).logs if otlp.HasField("logs") else [] + traces, _ = build_traces(decoded.spans, logs) + return traces + + +# ---------------------------------------------------------------- evaluation parameters + + +def eval_params_dict(params: evaluation_pb2.EvalParams) -> dict[str, Any]: + """The EvalParams fields that are set, under the Pydantic names; unset fields keep their defaults.""" + out: dict[str, Any] = {} + if params.evaluators: + out["evaluators"] = [_evaluator_dict(e) for e in params.evaluators] + for field in ("max_concurrent_traces", "max_concurrent_evals"): + if params.HasField(field): + out[field] = getattr(params, field) + if params.group_by in GROUP_BY_FROM_PB: + out["group_by"] = GROUP_BY_FROM_PB[params.group_by] + return out + + +def eval_params_from_pb(params: evaluation_pb2.EvalParams) -> EvalParams: + return EvalParams.model_validate(eval_params_dict(params)) + + +def _evaluator_dict(evaluator: evaluation_pb2.Evaluator) -> dict[str, Any]: + kind = evaluator.WhichOneof("kind") + if kind is None: + raise ApiError.invalid("each evaluator needs one of builtin, code or remote") + message = getattr(evaluator, kind) + out: dict[str, Any] = {"type": kind, "name": message.name} + if message.HasField("threshold"): + out["threshold"] = message.threshold + if kind == "builtin": + for field in ("judge_model", "credential_ref", "judge_base_url"): + if getattr(message, field): + out[field] = getattr(message, field) + if message.trajectory_match_type in TRAJECTORY_MATCH_FROM_PB: + out["trajectory_match_type"] = TRAJECTORY_MATCH_FROM_PB[message.trajectory_match_type] + return out + if message.HasField("timeout"): + out["timeout"] = message.timeout + if message.HasField("config"): + out["config"] = from_struct(message.config) + if message.executor: + out["executor"] = message.executor + if kind == "code": + out["path"] = message.path + else: + if message.source: + out["source"] = message.source + out["ref"] = message.ref + return out + + +def eval_params_to_pb(params: EvalParams | Mapping[str, Any]) -> evaluation_pb2.EvalParams: + if not isinstance(params, EvalParams): + params = _stored_eval_params(params) + return evaluation_pb2.EvalParams( + evaluators=[_evaluator_to_pb(e) for e in params.evaluators], + max_concurrent_traces=params.max_concurrent_traces, + max_concurrent_evals=params.max_concurrent_evals, + group_by=GROUP_BY.get(params.group_by, common_pb2.GROUP_BY_UNSPECIFIED), + ) + + +def _stored_eval_params(data: Mapping[str, Any]) -> EvalParams: + """Parameters of a stored run. Runs recorded by older versions may carry other keys.""" + try: + return EvalParams.model_validate({k: v for k, v in data.items() if k in EvalParams.model_fields}) + except ValidationError: + logger.warning("Stored eval_config does not validate; returning defaults") + return EvalParams() + + +def _evaluator_to_pb(evaluator: Any) -> evaluation_pb2.Evaluator: + if isinstance(evaluator, BuiltinMetricDef): + builtin = evaluation_pb2.BuiltinEvaluator( + name=evaluator.name, + judge_model=evaluator.judge_model or "", + trajectory_match_type=TRAJECTORY_MATCH.get(evaluator.trajectory_match_type or "", 0), + credential_ref=evaluator.credential_ref or "", + judge_base_url=evaluator.judge_base_url or "", + ) + if evaluator.threshold is not None: + builtin.threshold = evaluator.threshold + return evaluation_pb2.Evaluator(builtin=builtin) + common = { + "name": evaluator.name, + "threshold": evaluator.threshold, + "timeout": evaluator.timeout, + "config": to_struct(evaluator.config), + "executor": evaluator.executor, + } + if isinstance(evaluator, CodeEvaluatorDef): + return evaluation_pb2.Evaluator(code=evaluation_pb2.CodeEvaluator(path=evaluator.path, **common)) + if isinstance(evaluator, RemoteEvaluatorDef): + return evaluation_pb2.Evaluator( + remote=evaluation_pb2.RemoteEvaluator(source=evaluator.source, ref=evaluator.ref, **common) + ) + raise TypeError(f"unknown evaluator type {type(evaluator).__name__}") + + +# ---------------------------------------------------------------- evaluation results + + +def _invocations(scores: list[float | None], statuses: list[str]) -> list[evaluation_pb2.InvocationResult]: + out = [] + for score, status in itertools.zip_longest(scores, statuses): + result = evaluation_pb2.InvocationResult(status=EVAL_STATUS.get(status or "", 0)) + if _finite(score) is not None: + result.score = score + out.append(result) + return out + + +def metric_result_to_pb(result: MetricResult) -> evaluation_pb2.MetricResult: + out = evaluation_pb2.MetricResult( + metric_name=result.metric_name, + eval_status=EVAL_STATUS.get(result.eval_status, 0), + per_invocation=_invocations(result.per_invocation_scores, result.per_invocation_statuses), + ) + if _finite(result.score) is not None: + out.score = result.score + if result.error is not None: + out.error = result.error + if result.error_type is not None: + out.error_type = result.error_type + if result.details: + out.details.CopyFrom(to_struct(result.details)) + if _finite(result.duration_ms) is not None: + out.duration_ms = result.duration_ms + return out + + +def trace_result_to_pb(result: TraceResult) -> evaluation_pb2.TraceResult: + out = evaluation_pb2.TraceResult( + trace_id=result.trace_id, + num_invocations=result.num_invocations, + metric_results=[metric_result_to_pb(m) for m in result.metric_results], + conversion_warnings=result.conversion_warnings, + service_name=result.service_name or "", + agent_name=result.agent_name or "", + trace_ids=result.trace_ids, + conversation_id=result.conversation_id or "", + group_key=result.group_key or "", + eval_case_id=result.eval_case_id or "", + eval_case_match=result.eval_case_match or "", + turn_refs=[evaluation_pb2.TurnRef(trace_id=r.trace_id, span_id=r.span_id) for r in result.turn_refs], + ) + if result.performance_metrics: + out.performance_metrics.CopyFrom(to_struct(result.performance_metrics)) + return out + + +def run_result_to_pb(result: RunResult) -> evaluation_pb2.RunResult: + out = evaluation_pb2.RunResult( + trace_results=[trace_result_to_pb(t) for t in result.trace_results], + errors=result.errors, + run_id=result.run_id or "", + ) + if result.performance_metrics: + out.performance_metrics.CopyFrom(to_struct(result.performance_metrics)) + return out + + +# ---------------------------------------------------------------- runs + + +def run_spec_from_pb(spec: runs_pb2.RunSpec) -> RunSpec: + if spec.approach not in (runs_pb2.RUN_APPROACH_UNSPECIFIED, runs_pb2.RUN_APPROACH_TRACE_REPLAY): + raise ApiError.invalid("approach must be RUN_APPROACH_TRACE_REPLAY") + data: dict[str, Any] = { + "approach": "trace_replay", + "target": _target_dict(spec.target), + "eval_config": eval_params_from_pb(spec.eval_config).model_dump(by_alias=False), + "sinks": [from_struct(s) for s in spec.sinks], + "context": {"headers": dict(spec.context.headers)} if spec.context.headers else {}, + } + if spec.HasField("adk_eval_set"): + data["eval_set"] = from_struct(spec.adk_eval_set) + if spec.credential_refs: + data["credential_refs"] = {name: from_struct(ref) for name, ref in spec.credential_refs.items()} + return RunSpec.model_validate(data) + + +def _target_dict(target: runs_pb2.TraceTarget) -> dict[str, Any]: + kind = target.WhichOneof("kind") + if kind == "inline": + if len(target.inline.content) > MAX_DOCUMENT_BYTES: + raise ApiError.invalid("target.inline exceeds 10MB limit") + try: + document = json.loads(target.inline.content) + except (UnicodeDecodeError, ValueError) as exc: + raise ApiError.invalid("target.inline must be one JSON document") from exc + if not isinstance(document, dict): + raise ApiError.invalid("target.inline must be a JSON object") + return {"kind": "inline", "inline": document, "trace_format": TRACE_FORMAT_FROM_PB[target.inline.format]} + if kind == "otlp": + return {"kind": "inline", "inline": encode_traces(_otlp_traces(target.otlp)), "trace_format": "otlp-json"} + if kind == "http": + return { + "kind": "http", + "base_url": target.http.base_url, + "trace_id": target.http.trace_id, + "trace_format": TRACE_FORMAT_FROM_PB[target.http.format], + } + if kind == "uploaded": + raise ApiError.invalid("target.uploaded is output only") + raise ApiError.invalid("target needs one of inline, otlp or http") + + +def run_spec_to_pb(spec: RunSpec) -> runs_pb2.RunSpec: + target = spec.target + out = runs_pb2.RunSpec( + approach=runs_pb2.RUN_APPROACH_TRACE_REPLAY, + eval_config=eval_params_to_pb(spec.eval_config or {}), + sinks=[to_struct(s) for s in spec.sinks], + credential_refs={name: to_struct(ref) for name, ref in (spec.credential_refs or {}).items()}, + ) + trace_format = TRACE_FORMAT.get(target.trace_format or "", common_pb2.TRACE_FORMAT_UNSPECIFIED) + if target.kind == "inline": + content = json.dumps(target.inline or {}, separators=(",", ":")).encode() + out.target.inline.CopyFrom(common_pb2.TraceDocument(content=content, format=trace_format)) + elif target.kind == "http": + out.target.http.CopyFrom( + runs_pb2.HttpTraceTarget( + base_url=target.base_url or "", trace_id=target.trace_id or "", format=trace_format + ) + ) + else: + out.target.uploaded.CopyFrom( + runs_pb2.UploadedTraces(trace_count=target.trace_count or 0, trace_files=target.trace_files or []) + ) + if spec.eval_set is not None: + out.adk_eval_set.CopyFrom(to_struct(spec.eval_set)) + headers = spec.context.get("headers") if isinstance(spec.context, dict) else None + if isinstance(headers, dict): + out.context.headers.update({str(k): str(v) for k, v in headers.items()}) + return out + + +def run_to_pb(run: Run) -> runs_pb2.Run: + """The run as callers see it: redacted.""" + run = redact_run(run) + out = runs_pb2.Run( + run_id=str(run.run_id), + status=RUN_STATUS[run.status], + spec=run_spec_to_pb(run.spec), + attempt=run.attempt, + worker_id=run.worker_id or "", + error=run.error or "", + created_at=_timestamp(run.created_at), + started_at=_timestamp(run.started_at), + finished_at=_timestamp(run.finished_at), + cancel_requested=run.cancel_requested, + ) + if run.summary: + out.summary.CopyFrom(to_struct(run.summary)) + return out + + +def result_to_pb(result: Result) -> runs_pb2.Result: + out = runs_pb2.Result( + result_id=result.result_id, + run_id=str(result.run_id), + eval_set_item_id=result.eval_set_item_id, + eval_set_item_name=result.eval_set_item_name, + evaluator_name=result.evaluator_name, + evaluator_type=EVALUATOR_TYPE[result.evaluator_type], + status=RESULT_STATUS[result.status], + per_invocation=_invocations(result.per_invocation_scores, []), + trace_id=result.trace_id or "", + span_id=result.span_id or "", + error_text=result.error_text or "", + created_at=_timestamp(result.created_at), + ) + if _finite(result.score) is not None: + out.score = result.score + if result.details: + out.details.CopyFrom(to_struct(result.details)) + if result.tokens_used: + out.tokens_used.CopyFrom(to_struct(result.tokens_used)) + if result.latency_ms is not None: + out.latency_ms = result.latency_ms + return out diff --git a/src/agentevals/api/rpc/policy.py b/src/agentevals/api/rpc/policy.py new file mode 100644 index 00000000..b8f56801 --- /dev/null +++ b/src/agentevals/api/rpc/policy.py @@ -0,0 +1,152 @@ +"""Method policy and error mapping for the API's gRPC services. + +Every servicer method carries :func:`rpc`. The server refuses to start while a method of an +agentevals service has no policy, so a new RPC cannot ship without one. The wrapper checks the +method's requirement, then maps errors: an :class:`ApiError` to its status with a +``google.rpc.ErrorInfo``, a validation error to INVALID_ARGUMENT without the input values, and +anything else to INTERNAL with a fixed message, the details only in the server log. +""" + +from __future__ import annotations + +import functools +import inspect +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import grpc +from google.protobuf import any_pb2 +from google.rpc import code_pb2, error_details_pb2, status_pb2 +from grpc_status import rpc_status +from pydantic import ValidationError + +logger = logging.getLogger(__name__) + +ERROR_DOMAIN = "agentevals.api.v1alpha1" +REASON_STORAGE_NOT_CONFIGURED = "STORAGE_NOT_CONFIGURED" +REASON_RUN_SPEC_CONFLICT = "RUN_SPEC_CONFLICT" +REASON_INVALID_REQUEST = "INVALID_REQUEST" + +_POLICY_ATTR = "__agentevals_rpc_policy__" + + +class ApiError(Exception): + """An error the caller may see: a status code, a message and a stable reason.""" + + def __init__( + self, + code: grpc.StatusCode, + message: str, + reason: str | None = None, + metadata: dict[str, str] | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.reason = reason + self.metadata = metadata or {} + + @classmethod + def invalid(cls, message: str) -> ApiError: + return cls(grpc.StatusCode.INVALID_ARGUMENT, message, REASON_INVALID_REQUEST) + + @classmethod + def not_found(cls, message: str) -> ApiError: + return cls(grpc.StatusCode.NOT_FOUND, message) + + +@dataclass(frozen=True, slots=True) +class Policy: + # The method needs run storage; servicers expose it as ``run_service``. + storage: bool = False + + +def rpc(*, storage: bool = False) -> Callable: + """Declare a servicer method's policy and map its errors to gRPC statuses.""" + policy = Policy(storage=storage) + + def decorate(method: Callable) -> Callable: + name = method.__name__ + if inspect.isasyncgenfunction(method): + + @functools.wraps(method) + async def stream(self, request, context): + try: + _require(self, policy) + async for item in method(self, request, context): + yield item + except Exception as exc: + await _abort(context, name, exc) + + wrapper = stream + else: + + @functools.wraps(method) + async def unary(self, request, context): + try: + _require(self, policy) + return await method(self, request, context) + except Exception as exc: + await _abort(context, name, exc) + + wrapper = unary + setattr(wrapper, _POLICY_ATTR, policy) + return wrapper + + return decorate + + +def check_policies(servicer: Any, service: Any) -> None: + """Fail unless every method of ``service`` (a ServiceDescriptor) is decorated with :func:`rpc`.""" + missing = [m.name for m in service.methods if not hasattr(getattr(type(servicer), m.name, None), _POLICY_ATTR)] + if missing: + raise RuntimeError(f"{service.full_name} methods without an rpc policy: {', '.join(missing)}") + + +def _require(servicer: Any, policy: Policy) -> None: + if policy.storage and getattr(servicer, "run_service", None) is None: + raise ApiError( + grpc.StatusCode.FAILED_PRECONDITION, + "run storage is not configured (set AGENTEVALS_STORAGE_BACKEND=postgres)", + REASON_STORAGE_NOT_CONFIGURED, + ) + + +async def _abort(context: Any, method: str, exc: Exception) -> None: + if isinstance(exc, grpc.aio.AbortError): + raise exc + if isinstance(exc, ApiError): + status = _status(exc.code, exc.message, exc.reason, exc.metadata) + elif isinstance(exc, ValidationError): + status = _status(grpc.StatusCode.INVALID_ARGUMENT, validation_message(exc), REASON_INVALID_REQUEST) + else: + logger.exception("RPC %s failed", method) + status = _status(grpc.StatusCode.INTERNAL, "internal server error") + if status.code != grpc.StatusCode.INTERNAL: + logger.info("RPC %s returned %s", method, status.code.name) + await context.abort(status.code, status.details, status.trailing_metadata) + + +def validation_message(exc: ValidationError) -> str: + """Field paths and messages, never the input values, which can carry secrets.""" + problems = [ + f"{'.'.join(str(part) for part in error['loc']) or 'request'}: {error['msg']}" + for error in exc.errors(include_input=False, include_url=False, include_context=False) + ] + return "invalid request: " + "; ".join(problems) + + +def _status( + code: grpc.StatusCode, + message: str, + reason: str | None = None, + metadata: dict[str, str] | None = None, +) -> grpc.Status: + proto = status_pb2.Status(code=getattr(code_pb2, code.name), message=message) + if reason: + detail = any_pb2.Any() + detail.Pack(error_details_pb2.ErrorInfo(reason=reason, domain=ERROR_DOMAIN, metadata=metadata or {})) + proto.details.append(detail) + return rpc_status.to_status(proto) diff --git a/src/agentevals/api/rpc/server.py b/src/agentevals/api/rpc/server.py new file mode 100644 index 00000000..8c5f369f --- /dev/null +++ b/src/agentevals/api/rpc/server.py @@ -0,0 +1,100 @@ +"""The API's native gRPC server: the v1alpha1 services, ``grpc.health.v1``, optional reflection and +opt in tracing with the upstream gRPC instrumentation. + +A browser cannot speak native gRPC (no HTTP/2 trailers from ``fetch``), so this listener is out of +reach of web pages. Every caller that can reach it is trusted: expose it only to trusted callers. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +import grpc +from grpc_health.v1 import health, health_pb2, health_pb2_grpc + +from ...telemetry import api_tracer_provider, env_flag +from ..otlp_grpc import listen_address +from .policy import check_policies +from .services import ApiContext, Ports, servicers + +logger = logging.getLogger(__name__) + +REFLECTION_ENV = "AGENTEVALS_GRPC_REFLECTION" +MAX_MESSAGE_BYTES = 16 * 1024 * 1024 +MAX_CONCURRENT_RPCS = 64 +SHUTDOWN_GRACE_SECONDS = 5 + +SERVER_OPTIONS = [ + ("grpc.max_receive_message_length", MAX_MESSAGE_BYTES), + ("grpc.max_send_message_length", MAX_MESSAGE_BYTES), + # Watch streams stay open for hours; pings keep idle connections through load balancers. + ("grpc.keepalive_time_ms", 60_000), + ("grpc.keepalive_timeout_ms", 20_000), + ("grpc.http2.min_ping_interval_without_data_ms", 10_000), +] + + +@dataclass(frozen=True, slots=True) +class ApiGrpcSettings: + host: str + port: int + ports: Ports + + +@dataclass(slots=True) +class ApiServer: + server: grpc.aio.Server + health: health.aio.HealthServicer + tracer_provider: Any | None + port: int + + +async def start_api_server(context: ApiContext, host: str, port: int) -> ApiServer: + tracer_provider = api_tracer_provider() + interceptors = [] + if tracer_provider is not None: + from opentelemetry.instrumentation.grpc import aio_server_interceptor, filters + + noise = filters.any_of(filters.health_check(), filters.service_prefix("grpc.reflection.")) + interceptors.append(aio_server_interceptor(tracer_provider=tracer_provider, filter_=filters.negate(noise))) + + server = grpc.aio.server( + interceptors=interceptors, options=SERVER_OPTIONS, maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS + ) + names = [] + for servicer, register, service in servicers(context): + check_policies(servicer, service) + register(servicer, server) + names.append(service.full_name) + + health_servicer = health.aio.HealthServicer() + health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server) + for name in ["", *names]: + await health_servicer.set(name, health_pb2.HealthCheckResponse.SERVING) + + if env_flag(REFLECTION_ENV): + from grpc_reflection.v1alpha import reflection + + reflection.enable_server_reflection([*names, health.SERVICE_NAME, reflection.SERVICE_NAME], server) + + address = listen_address(host, port) + bound = server.add_insecure_port(address) + if bound == 0: + raise RuntimeError(f"Failed to bind the API gRPC server to {address}") + await server.start() + logger.info( + "API gRPC server on %s (reflection %s, tracing %s)", + listen_address(host, bound), + "on" if env_flag(REFLECTION_ENV) else "off", + "on" if tracer_provider is not None else "off", + ) + return ApiServer(server=server, health=health_servicer, tracer_provider=tracer_provider, port=bound) + + +async def stop_api_server(api: ApiServer) -> None: + await api.health.enter_graceful_shutdown() + await api.server.stop(grace=SHUTDOWN_GRACE_SECONDS) + if api.tracer_provider is not None: + api.tracer_provider.shutdown() diff --git a/src/agentevals/api/rpc/services.py b/src/agentevals/api/rpc/services.py new file mode 100644 index 00000000..5df7492e --- /dev/null +++ b/src/agentevals/api/rpc/services.py @@ -0,0 +1,401 @@ +"""The five v1alpha1 services. Each method only translates: the work happens in the same code the +REST API and the CLI use.""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import json +import logging +import os +import time +from dataclasses import dataclass +from datetime import datetime +from typing import Any +from uuid import UUID + +import grpc + +from ... import __version__ +from ...adk_bridge import eval_set_from_turns, load_eval_set_from_dict +from ...builtin_metrics import metric_catalog +from ...genai.extract import extract_conversation +from ...run.service import RunService, RunSubmitConflict +from ...runner import evaluation_groups +from ...storage.models import RunStatus +from ...streaming.manager import LiveManager, TooManyWatchers +from ..evaluation import ( + CredentialResolutionError, + Done, + Evaluation, + EvaluationEvent, + GroupResult, + Performance, + Progress, + stream_evaluation, +) +from ..v1alpha1 import ( + common_pb2, + conversion_pb2, + conversion_pb2_grpc, + evaluation_pb2, + evaluation_pb2_grpc, + runs_pb2, + runs_pb2_grpc, + sessions_pb2, + sessions_pb2_grpc, + system_pb2, + system_pb2_grpc, +) +from ..views import conversation_view +from . import convert +from .policy import REASON_RUN_SPEC_CONFLICT, ApiError, rpc, validation_message + +logger = logging.getLogger(__name__) + +DEFAULT_PAGE_SIZE = 50 +MAX_PAGE_SIZE = 100 +WATCH_RUN_POLL_SECONDS = 1.0 +# A watch ends after this long even when the run has not finished; the caller watches again. +WATCH_RUN_MAX_SECONDS = 3600.0 +TERMINAL_RUN_STATUSES = frozenset({RunStatus.SUCCEEDED, RunStatus.FAILED, RunStatus.CANCELLED}) + + +@dataclass(frozen=True, slots=True) +class Ports: + http: int = 0 + otlp_http: int = 0 + otlp_grpc: int = 0 + api_grpc: int = 0 + + +@dataclass(slots=True) +class ApiContext: + manager: LiveManager + run_service: RunService | None = None + ports: Ports = Ports() + + +# ---------------------------------------------------------------- paging + + +def _page_size(page: common_pb2.PageRequest) -> int: + if page.limit < 0 or page.limit > MAX_PAGE_SIZE: + raise ApiError.invalid(f"page.limit must be between 0 and {MAX_PAGE_SIZE}") + return page.limit or DEFAULT_PAGE_SIZE + + +def _token(data: dict[str, Any]) -> str: + return base64.urlsafe_b64encode(json.dumps(data, separators=(",", ":")).encode()).decode() + + +def _from_token(token: str) -> dict[str, Any]: + try: + data = json.loads(base64.urlsafe_b64decode(token.encode())) + except ValueError as exc: + raise ApiError.invalid("page.page_token is not valid") from exc + if not isinstance(data, dict): + raise ApiError.invalid("page.page_token is not valid") + return data + + +def _uuid(value: str, field: str) -> UUID: + try: + return UUID(value) + except ValueError as exc: + raise ApiError.invalid(f"{field} must be a UUID") from exc + + +# ---------------------------------------------------------------- sessions + + +class SessionServicer(sessions_pb2_grpc.SessionServiceServicer): + def __init__(self, context: ApiContext) -> None: + self._manager = context.manager + + @rpc() + async def ListSessions(self, request, context): # noqa: N802 + size = _page_size(request.page) + ordered = sorted(self._manager.sessions.values(), key=lambda s: (s.started_at, s.session_id), reverse=True) + if request.page.page_token: + after = _from_token(request.page.page_token) + last = (str(after.get("t", "")), str(after.get("id", ""))) + ordered = [s for s in ordered if (s.started_at.isoformat(), s.session_id) < last] + page = ordered[:size] + response = sessions_pb2.ListSessionsResponse( + sessions=[self._manager.view(s).summary_only.upsert.summary for s in page] + ) + if len(ordered) > size: + response.page.next_page_token = _token({"t": page[-1].started_at.isoformat(), "id": page[-1].session_id}) + return response + + @rpc() + async def GetSession(self, request, context): # noqa: N802 + session = self._manager.sessions.get(request.id) + if session is None: + raise ApiError.not_found(f"session {request.id} not found") + return sessions_pb2.GetSessionResponse(session=self._manager.view(session).with_turns.upsert) + + @rpc() + async def WatchSessions(self, request, context): # noqa: N802 + try: + watcher = self._manager.subscribe(include_turns=request.include_turns) + except TooManyWatchers as exc: + raise ApiError(grpc.StatusCode.RESOURCE_EXHAUSTED, str(exc)) from exc + try: + while (event := await watcher.next()) is not None: + yield watcher.response(event) + finally: + self._manager.unsubscribe(watcher) + + +# ---------------------------------------------------------------- evaluation + + +def _evaluation_response(event: EvaluationEvent) -> evaluation_pb2.EvaluateTracesResponse: + if isinstance(event, Progress): + return evaluation_pb2.EvaluateTracesResponse(progress=evaluation_pb2.EvaluationProgress(message=event.message)) + if isinstance(event, Performance): + return evaluation_pb2.EvaluateTracesResponse( + performance=evaluation_pb2.GroupPerformance( + trace_id=event.trace_id, + performance_metrics=convert.to_struct(event.performance_metrics), + trace_metadata=convert.to_struct(event.trace_metadata), + ) + ) + if isinstance(event, GroupResult): + return evaluation_pb2.EvaluateTracesResponse(trace_result=convert.trace_result_to_pb(event.result)) + assert isinstance(event, Done) + return evaluation_pb2.EvaluateTracesResponse(done=convert.run_result_to_pb(event.result)) + + +class EvaluationServicer(evaluation_pb2_grpc.EvaluationServiceServicer): + def __init__(self, context: ApiContext) -> None: + self._context = context + + @rpc() + async def ListMetrics(self, request, context): # noqa: N802 + return evaluation_pb2.ListMetricsResponse( + metrics=[evaluation_pb2.Metric(**dataclasses.asdict(m)) for m in metric_catalog()] + ) + + @rpc() + async def EvaluateTraces(self, request, context): # noqa: N802 + traces, load_errors = await asyncio.to_thread(convert.load_trace_input, request.traces) + eval_set_dict = convert.from_struct(request.adk_eval_set) if request.HasField("adk_eval_set") else None + evaluation = Evaluation( + traces=traces, + params=convert.eval_params_from_pb(request.params), + eval_set=load_eval_set_from_dict(eval_set_dict) if eval_set_dict is not None else None, + credential_refs={name: convert.from_struct(ref) for name, ref in request.credential_refs.items()} or None, + load_errors=load_errors, + eval_set_dict=eval_set_dict, + ) + try: + async for event in stream_evaluation(evaluation, run_service=self._context.run_service): + yield _evaluation_response(event) + except CredentialResolutionError as exc: + logger.info("EvaluateTraces: %s", exc) + raise ApiError(grpc.StatusCode.FAILED_PRECONDITION, "a credential reference could not be resolved") from exc + + +# ---------------------------------------------------------------- runs + + +class RunServicer(runs_pb2_grpc.RunServiceServicer): + def __init__(self, context: ApiContext) -> None: + self._context = context + + @property + def run_service(self) -> RunService | None: + return self._context.run_service + + async def _run(self, run_id: str): + run = await self.run_service.get(_uuid(run_id, "run_id")) + if run is None: + raise ApiError.not_found(f"run {run_id} not found") + return run + + @rpc(storage=True) + async def SubmitRun(self, request, context): # noqa: N802 + run_id = _uuid(request.run_id, "run_id") if request.run_id else None + spec = convert.run_spec_from_pb(request.spec) + try: + run = await self.run_service.submit(run_id=run_id, spec=spec) + except RunSubmitConflict as exc: + raise ApiError( + grpc.StatusCode.ALREADY_EXISTS, + "run_id already exists with a different spec", + REASON_RUN_SPEC_CONFLICT, + {"run_id": str(exc.persisted.run_id)}, + ) from exc + except ValueError as exc: + cause = exc.__cause__ + message = validation_message(cause) if hasattr(cause, "errors") else str(exc) + raise ApiError.invalid(message) from exc + return runs_pb2.SubmitRunResponse(run=convert.run_to_pb(run)) + + @rpc(storage=True) + async def GetRun(self, request, context): # noqa: N802 + return runs_pb2.GetRunResponse(run=convert.run_to_pb(await self._run(request.run_id))) + + @rpc(storage=True) + async def ListRuns(self, request, context): # noqa: N802 + size = _page_size(request.page) + before = None + if request.page.page_token: + try: + before = datetime.fromisoformat(str(_from_token(request.page.page_token).get("before", ""))) + except ValueError as exc: + raise ApiError.invalid("page.page_token is not valid") from exc + statuses = [convert.RUN_STATUS_FROM_PB[s] for s in request.statuses if s in convert.RUN_STATUS_FROM_PB] + runs = await self.run_service.list(status=statuses or None, limit=size + 1, before=before) + response = runs_pb2.ListRunsResponse(runs=[convert.run_to_pb(r) for r in runs[:size]]) + if len(runs) > size: + response.page.next_page_token = _token({"before": runs[size - 1].created_at.isoformat()}) + return response + + @rpc(storage=True) + async def CancelRun(self, request, context): # noqa: N802 + run_id = _uuid(request.run_id, "run_id") + await self.run_service.cancel(run_id) + return runs_pb2.CancelRunResponse(run=convert.run_to_pb(await self._run(request.run_id))) + + @rpc(storage=True) + async def ListRunResults(self, request, context): # noqa: N802 + size = _page_size(request.page) + run = await self._run(request.run_id) + offset = int(_from_token(request.page.page_token).get("offset", 0)) if request.page.page_token else 0 + results = await self.run_service.list_results(run.run_id) + page = results[offset : offset + size] + response = runs_pb2.ListRunResultsResponse(results=[convert.result_to_pb(r) for r in page]) + if offset + size < len(results): + response.page.next_page_token = _token({"offset": offset + size}) + return response + + @rpc(storage=True) + async def WatchRun(self, request, context): # noqa: N802 + run = await self._run(request.run_id) + seen: dict[str, dict[str, Any]] = {} + last_state = None + deadline = time.monotonic() + WATCH_RUN_MAX_SECONDS + while True: + results = await self.run_service.list_results(run.run_id) + changed = [] + for result in results: + content = result.model_dump(mode="json") + if seen.get(result.result_id) != content: + seen[result.result_id] = content + changed.append(result) + state = (run.status, run.attempt, run.error, run.finished_at, run.cancel_requested) + if state != last_state or changed: + last_state = state + yield runs_pb2.WatchRunResponse( + run=convert.run_to_pb(run), results=[convert.result_to_pb(r) for r in changed] + ) + if run.status in TERMINAL_RUN_STATUSES or time.monotonic() > deadline: + return + await asyncio.sleep(WATCH_RUN_POLL_SECONDS) + run = await self._run(request.run_id) + + +# ---------------------------------------------------------------- conversion + + +def _convert(traces, group_by: str) -> list[conversion_pb2.ConvertedConversation]: + out = [] + for key, members in evaluation_groups(traces, group_by): + conversation = extract_conversation(members, key) + spans = {(s.trace_id, s.span_id): s for t in members for s in t.spans.values()} + out.append( + conversion_pb2.ConvertedConversation( + conversation=conversation_view(conversation, spans), + golden_adk_eval_set=convert.to_struct(eval_set_from_turns(key, conversation.turns)), + ) + ) + return out + + +class ConversionServicer(conversion_pb2_grpc.ConversionServiceServicer): + def __init__(self, context: ApiContext) -> None: + pass + + @rpc() + async def ConvertTraces(self, request, context): # noqa: N802 + traces, warnings = await asyncio.to_thread(convert.load_trace_input, request.traces) + group_by = convert.GROUP_BY_FROM_PB.get(request.group_by, "trace") + conversations = await asyncio.to_thread(_convert, traces, group_by) + return conversion_pb2.ConvertTracesResponse(conversations=conversations, warnings=warnings) + + +# ---------------------------------------------------------------- system + + +class SystemServicer(system_pb2_grpc.SystemServiceServicer): + def __init__(self, context: ApiContext) -> None: + self._context = context + + @rpc() + async def GetVersion(self, request, context): # noqa: N802 + return system_pb2.GetVersionResponse(version=__version__) + + @rpc() + async def GetConfig(self, request, context): # noqa: N802 + from ... import evaluation_events + + ports, limits = self._context.ports, self._context.manager.store.limits + response = system_pb2.GetConfigResponse( + ports=system_pb2.Ports( + http=ports.http, otlp_http=ports.otlp_http, otlp_grpc=ports.otlp_grpc, api_grpc=ports.api_grpc + ), + live_limits=system_pb2.LiveLimits( + max_sessions=limits.max_sessions, + spans_per_trace=limits.spans_per_trace, + spans_per_session=limits.spans_per_session, + traces_per_session=limits.traces_per_session, + max_bytes=limits.max_bytes, + ), + runs_enabled=self._context.run_service is not None, + evaluation_events_enabled=evaluation_events.enabled(), + judge_keys=system_pb2.JudgeKeys( + google=bool(os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")), + anthropic=bool(os.environ.get("ANTHROPIC_API_KEY")), + openai=bool(os.environ.get("OPENAI_API_KEY")), + ), + ) + response.live_limits.completion_grace.FromNanoseconds(int(limits.completion_grace_seconds * 1e9)) + response.live_limits.idle_timeout.FromNanoseconds(int(limits.idle_timeout_seconds * 1e9)) + response.live_limits.session_ttl.FromNanoseconds(int(limits.session_ttl_seconds * 1e9)) + return response + + +def servicers(context: ApiContext) -> list[tuple[Any, Any, Any]]: + """Every service as (servicer, registration function, service descriptor).""" + return [ + ( + SessionServicer(context), + sessions_pb2_grpc.add_SessionServiceServicer_to_server, + sessions_pb2.DESCRIPTOR.services_by_name["SessionService"], + ), + ( + EvaluationServicer(context), + evaluation_pb2_grpc.add_EvaluationServiceServicer_to_server, + evaluation_pb2.DESCRIPTOR.services_by_name["EvaluationService"], + ), + ( + RunServicer(context), + runs_pb2_grpc.add_RunServiceServicer_to_server, + runs_pb2.DESCRIPTOR.services_by_name["RunService"], + ), + ( + ConversionServicer(context), + conversion_pb2_grpc.add_ConversionServiceServicer_to_server, + conversion_pb2.DESCRIPTOR.services_by_name["ConversionService"], + ), + ( + SystemServicer(context), + system_pb2_grpc.add_SystemServiceServicer_to_server, + system_pb2.DESCRIPTOR.services_by_name["SystemService"], + ), + ] diff --git a/src/agentevals/api/runs_routes.py b/src/agentevals/api/runs_routes.py index 7318ade8..555070d1 100644 --- a/src/agentevals/api/runs_routes.py +++ b/src/agentevals/api/runs_routes.py @@ -25,6 +25,7 @@ from ..run.service import RunService, RunSubmitConflict from ..storage.models import Result, Run, RunSpec, RunStatus from .models import CamelModel, StandardResponse +from .redact import redact_run logger = logging.getLogger(__name__) @@ -83,10 +84,10 @@ async def submit_run(payload: RunRequest, request: Request): status_code=status.HTTP_409_CONFLICT, detail={ "message": "run_id already exists with a different spec", - "persisted": exc.persisted.model_dump(mode="json", by_alias=True), + "persisted": redact_run(exc.persisted).model_dump(mode="json", by_alias=True), }, ) from exc - return StandardResponse(data=run) + return StandardResponse(data=redact_run(run)) @runs_router.get("/runs/{run_id}", response_model=StandardResponse[Run]) @@ -95,7 +96,7 @@ async def get_run(run_id: UUID, request: Request): run = await service.get(run_id) if run is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"run {run_id} not found") - return StandardResponse(data=run) + return StandardResponse(data=redact_run(run)) @runs_router.get("/runs", response_model=StandardResponse[list[Run]]) @@ -107,7 +108,7 @@ async def list_runs( ): service = _service(request) runs = await service.list(status=status_filter, limit=limit, before=before) - return StandardResponse(data=runs) + return StandardResponse(data=[redact_run(r) for r in runs]) @runs_router.get("/runs/{run_id}/results", response_model=StandardResponse[list[Result]]) @@ -129,4 +130,4 @@ async def cancel_run(run_id: UUID, request: Request): run = await service.get(run_id) if run is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"run {run_id} not found") - return StandardResponse(data=run) + return StandardResponse(data=redact_run(run)) diff --git a/src/agentevals/api/streaming_routes.py b/src/agentevals/api/streaming_routes.py index 3217febf..2df950c7 100644 --- a/src/agentevals/api/streaming_routes.py +++ b/src/agentevals/api/streaming_routes.py @@ -96,9 +96,7 @@ async def _do_create_eval_set( @streaming_router.get("/sessions", response_model=StandardResponse[list[SessionInfo]]) async def list_sessions(manager: LiveManager = Depends(require_trace_manager)): - return StandardResponse( - data=[manager.session_info(session, with_invocations=True) for session in manager.sessions.values()] - ) + return StandardResponse(data=[manager.session_info(session) for session in manager.sessions.values()]) @streaming_router.post("/create-eval-set", response_model=StandardResponse[CreateEvalSetData]) diff --git a/src/agentevals/api/v1alpha1/common_pb2.py b/src/agentevals/api/v1alpha1/common_pb2.py new file mode 100644 index 00000000..dbaea544 --- /dev/null +++ b/src/agentevals/api/v1alpha1/common_pb2.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/common.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/common.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from opentelemetry.proto.logs.v1 import logs_pb2 as opentelemetry_dot_proto_dot_logs_dot_v1_dot_logs__pb2 +from opentelemetry.proto.trace.v1 import trace_pb2 as opentelemetry_dot_proto_dot_trace_dot_v1_dot_trace__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$agentevals/api/v1alpha1/common.proto\x12\x17\x61gentevals.api.v1alpha1\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\"B\n\x0bPageRequest\x12\x14\n\x05limit\x18\x01 \x01(\x05R\x05limit\x12\x1d\n\npage_token\x18\x02 \x01(\tR\tpageToken\"6\n\x0cPageResponse\x12&\n\x0fnext_page_token\x18\x01 \x01(\tR\rnextPageToken\"{\n\rTraceDocument\x12\x18\n\x07\x63ontent\x18\x01 \x01(\x0cR\x07\x63ontent\x12<\n\x06\x66ormat\x18\x02 \x01(\x0e\x32$.agentevals.api.v1alpha1.TraceFormatR\x06\x66ormat\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\"\x8c\x01\n\rOtlpTelemetry\x12@\n\x06traces\x18\x01 \x01(\x0b\x32(.opentelemetry.proto.trace.v1.TracesDataR\x06traces\x12\x39\n\x04logs\x18\x02 \x01(\x0b\x32%.opentelemetry.proto.logs.v1.LogsDataR\x04logs\"\x8e\x01\n\nTraceInput\x12:\n\x04otlp\x18\x01 \x01(\x0b\x32&.agentevals.api.v1alpha1.OtlpTelemetryR\x04otlp\x12\x44\n\tdocuments\x18\x02 \x03(\x0b\x32&.agentevals.api.v1alpha1.TraceDocumentR\tdocuments*\x82\x01\n\x0bTraceFormat\x12\x1c\n\x18TRACE_FORMAT_UNSPECIFIED\x10\x00\x12\x1a\n\x16TRACE_FORMAT_OTLP_JSON\x10\x01\x12\x1b\n\x17TRACE_FORMAT_OTLP_JSONL\x10\x02\x12\x1c\n\x18TRACE_FORMAT_JAEGER_JSON\x10\x03*e\n\x07GroupBy\x12\x18\n\x14GROUP_BY_UNSPECIFIED\x10\x00\x12\x11\n\rGROUP_BY_AUTO\x10\x01\x12\x12\n\x0eGROUP_BY_TRACE\x10\x02\x12\x19\n\x15GROUP_BY_CONVERSATION\x10\x03\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.common_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_TRACEFORMAT']._serialized_start=685 + _globals['_TRACEFORMAT']._serialized_end=815 + _globals['_GROUPBY']._serialized_start=817 + _globals['_GROUPBY']._serialized_end=918 + _globals['_PAGEREQUEST']._serialized_start=147 + _globals['_PAGEREQUEST']._serialized_end=213 + _globals['_PAGERESPONSE']._serialized_start=215 + _globals['_PAGERESPONSE']._serialized_end=269 + _globals['_TRACEDOCUMENT']._serialized_start=271 + _globals['_TRACEDOCUMENT']._serialized_end=394 + _globals['_OTLPTELEMETRY']._serialized_start=397 + _globals['_OTLPTELEMETRY']._serialized_end=537 + _globals['_TRACEINPUT']._serialized_start=540 + _globals['_TRACEINPUT']._serialized_end=682 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/common_pb2.pyi b/src/agentevals/api/v1alpha1/common_pb2.pyi new file mode 100644 index 00000000..079d4ac8 --- /dev/null +++ b/src/agentevals/api/v1alpha1/common_pb2.pyi @@ -0,0 +1,72 @@ +from opentelemetry.proto.logs.v1 import logs_pb2 as _logs_pb2 +from opentelemetry.proto.trace.v1 import trace_pb2 as _trace_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class TraceFormat(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + TRACE_FORMAT_UNSPECIFIED: _ClassVar[TraceFormat] + TRACE_FORMAT_OTLP_JSON: _ClassVar[TraceFormat] + TRACE_FORMAT_OTLP_JSONL: _ClassVar[TraceFormat] + TRACE_FORMAT_JAEGER_JSON: _ClassVar[TraceFormat] + +class GroupBy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + GROUP_BY_UNSPECIFIED: _ClassVar[GroupBy] + GROUP_BY_AUTO: _ClassVar[GroupBy] + GROUP_BY_TRACE: _ClassVar[GroupBy] + GROUP_BY_CONVERSATION: _ClassVar[GroupBy] +TRACE_FORMAT_UNSPECIFIED: TraceFormat +TRACE_FORMAT_OTLP_JSON: TraceFormat +TRACE_FORMAT_OTLP_JSONL: TraceFormat +TRACE_FORMAT_JAEGER_JSON: TraceFormat +GROUP_BY_UNSPECIFIED: GroupBy +GROUP_BY_AUTO: GroupBy +GROUP_BY_TRACE: GroupBy +GROUP_BY_CONVERSATION: GroupBy + +class PageRequest(_message.Message): + __slots__ = ("limit", "page_token") + LIMIT_FIELD_NUMBER: _ClassVar[int] + PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] + limit: int + page_token: str + def __init__(self, limit: _Optional[int] = ..., page_token: _Optional[str] = ...) -> None: ... + +class PageResponse(_message.Message): + __slots__ = ("next_page_token",) + NEXT_PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] + next_page_token: str + def __init__(self, next_page_token: _Optional[str] = ...) -> None: ... + +class TraceDocument(_message.Message): + __slots__ = ("content", "format", "name") + CONTENT_FIELD_NUMBER: _ClassVar[int] + FORMAT_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + content: bytes + format: TraceFormat + name: str + def __init__(self, content: _Optional[bytes] = ..., format: _Optional[_Union[TraceFormat, str]] = ..., name: _Optional[str] = ...) -> None: ... + +class OtlpTelemetry(_message.Message): + __slots__ = ("traces", "logs") + TRACES_FIELD_NUMBER: _ClassVar[int] + LOGS_FIELD_NUMBER: _ClassVar[int] + traces: _trace_pb2.TracesData + logs: _logs_pb2.LogsData + def __init__(self, traces: _Optional[_Union[_trace_pb2.TracesData, _Mapping]] = ..., logs: _Optional[_Union[_logs_pb2.LogsData, _Mapping]] = ...) -> None: ... + +class TraceInput(_message.Message): + __slots__ = ("otlp", "documents") + OTLP_FIELD_NUMBER: _ClassVar[int] + DOCUMENTS_FIELD_NUMBER: _ClassVar[int] + otlp: OtlpTelemetry + documents: _containers.RepeatedCompositeFieldContainer[TraceDocument] + def __init__(self, otlp: _Optional[_Union[OtlpTelemetry, _Mapping]] = ..., documents: _Optional[_Iterable[_Union[TraceDocument, _Mapping]]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/common_pb2_grpc.py b/src/agentevals/api/v1alpha1/common_pb2_grpc.py new file mode 100644 index 00000000..2daafffe --- /dev/null +++ b/src/agentevals/api/v1alpha1/common_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/src/agentevals/api/v1alpha1/conversion_pb2.py b/src/agentevals/api/v1alpha1/conversion_pb2.py new file mode 100644 index 00000000..ccfd7cb0 --- /dev/null +++ b/src/agentevals/api/v1alpha1/conversion_pb2.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/conversion.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/conversion.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agentevals.api.v1alpha1 import common_pb2 as agentevals_dot_api_dot_v1alpha1_dot_common__pb2 +from agentevals.api.v1alpha1 import views_pb2 as agentevals_dot_api_dot_v1alpha1_dot_views__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(agentevals/api/v1alpha1/conversion.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a#agentevals/api/v1alpha1/views.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x90\x01\n\x14\x43onvertTracesRequest\x12;\n\x06traces\x18\x01 \x01(\x0b\x32#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n\x08group_by\x18\x02 \x01(\x0e\x32 .agentevals.api.v1alpha1.GroupByR\x07groupBy\"\xae\x01\n\x15\x43onvertedConversation\x12M\n\x0c\x63onversation\x18\x01 \x01(\x0b\x32).agentevals.api.v1alpha1.ConversationViewR\x0c\x63onversation\x12\x46\n\x13golden_adk_eval_set\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x10goldenAdkEvalSet\"\x89\x01\n\x15\x43onvertTracesResponse\x12T\n\rconversations\x18\x01 \x03(\x0b\x32..agentevals.api.v1alpha1.ConvertedConversationR\rconversations\x12\x1a\n\x08warnings\x18\x02 \x03(\tR\x08warnings2\x83\x01\n\x11\x43onversionService\x12n\n\rConvertTraces\x12-.agentevals.api.v1alpha1.ConvertTracesRequest\x1a..agentevals.api.v1alpha1.ConvertTracesResponseBQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.conversion_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_CONVERTTRACESREQUEST']._serialized_start=175 + _globals['_CONVERTTRACESREQUEST']._serialized_end=319 + _globals['_CONVERTEDCONVERSATION']._serialized_start=322 + _globals['_CONVERTEDCONVERSATION']._serialized_end=496 + _globals['_CONVERTTRACESRESPONSE']._serialized_start=499 + _globals['_CONVERTTRACESRESPONSE']._serialized_end=636 + _globals['_CONVERSIONSERVICE']._serialized_start=639 + _globals['_CONVERSIONSERVICE']._serialized_end=770 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/conversion_pb2.pyi b/src/agentevals/api/v1alpha1/conversion_pb2.pyi new file mode 100644 index 00000000..ee3efdaa --- /dev/null +++ b/src/agentevals/api/v1alpha1/conversion_pb2.pyi @@ -0,0 +1,34 @@ +from agentevals.api.v1alpha1 import common_pb2 as _common_pb2 +from agentevals.api.v1alpha1 import views_pb2 as _views_pb2 +from google.protobuf import struct_pb2 as _struct_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class ConvertTracesRequest(_message.Message): + __slots__ = ("traces", "group_by") + TRACES_FIELD_NUMBER: _ClassVar[int] + GROUP_BY_FIELD_NUMBER: _ClassVar[int] + traces: _common_pb2.TraceInput + group_by: _common_pb2.GroupBy + def __init__(self, traces: _Optional[_Union[_common_pb2.TraceInput, _Mapping]] = ..., group_by: _Optional[_Union[_common_pb2.GroupBy, str]] = ...) -> None: ... + +class ConvertedConversation(_message.Message): + __slots__ = ("conversation", "golden_adk_eval_set") + CONVERSATION_FIELD_NUMBER: _ClassVar[int] + GOLDEN_ADK_EVAL_SET_FIELD_NUMBER: _ClassVar[int] + conversation: _views_pb2.ConversationView + golden_adk_eval_set: _struct_pb2.Struct + def __init__(self, conversation: _Optional[_Union[_views_pb2.ConversationView, _Mapping]] = ..., golden_adk_eval_set: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + +class ConvertTracesResponse(_message.Message): + __slots__ = ("conversations", "warnings") + CONVERSATIONS_FIELD_NUMBER: _ClassVar[int] + WARNINGS_FIELD_NUMBER: _ClassVar[int] + conversations: _containers.RepeatedCompositeFieldContainer[ConvertedConversation] + warnings: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, conversations: _Optional[_Iterable[_Union[ConvertedConversation, _Mapping]]] = ..., warnings: _Optional[_Iterable[str]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/conversion_pb2_grpc.py b/src/agentevals/api/v1alpha1/conversion_pb2_grpc.py new file mode 100644 index 00000000..68c2d1ea --- /dev/null +++ b/src/agentevals/api/v1alpha1/conversion_pb2_grpc.py @@ -0,0 +1,78 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agentevals.api.v1alpha1 import conversion_pb2 as agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2 + + +class ConversionServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ConvertTraces = channel.unary_unary( + '/agentevals.api.v1alpha1.ConversionService/ConvertTraces', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesResponse.FromString, + _registered_method=True) + + +class ConversionServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ConvertTraces(self, request, context): + """Reads traces into conversations without evaluating them. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ConversionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ConvertTraces': grpc.unary_unary_rpc_method_handler( + servicer.ConvertTraces, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agentevals.api.v1alpha1.ConversionService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agentevals.api.v1alpha1.ConversionService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class ConversionService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ConvertTraces(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.ConversionService/ConvertTraces', + agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_conversion__pb2.ConvertTracesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/agentevals/api/v1alpha1/evaluation_pb2.py b/src/agentevals/api/v1alpha1/evaluation_pb2.py new file mode 100644 index 00000000..821d2cce --- /dev/null +++ b/src/agentevals/api/v1alpha1/evaluation_pb2.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/evaluation.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/evaluation.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agentevals.api.v1alpha1 import common_pb2 as agentevals_dot_api_dot_v1alpha1_dot_common__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(agentevals/api/v1alpha1/evaluation.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x14\n\x12ListMetricsRequest\"\x91\x02\n\x06Metric\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n\x08\x63\x61tegory\x18\x02 \x01(\tR\x08\x63\x61tegory\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12*\n\x11requires_eval_set\x18\x04 \x01(\x08R\x0frequiresEvalSet\x12!\n\x0crequires_llm\x18\x05 \x01(\x08R\x0brequiresLlm\x12!\n\x0crequires_gcp\x18\x06 \x01(\x08R\x0brequiresGcp\x12)\n\x10requires_rubrics\x18\x07 \x01(\x08R\x0frequiresRubrics\x12\x18\n\x07working\x18\x08 \x01(\x08R\x07working\"P\n\x13ListMetricsResponse\x12\x39\n\x07metrics\x18\x01 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.MetricR\x07metrics\"\xa7\x02\n\x10\x42uiltinEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1f\n\x0bjudge_model\x18\x03 \x01(\tR\njudgeModel\x12`\n\x15trajectory_match_type\x18\x04 \x01(\x0e\x32,.agentevals.api.v1alpha1.TrajectoryMatchTypeR\x13trajectoryMatchType\x12%\n\x0e\x63redential_ref\x18\x05 \x01(\tR\rcredentialRef\x12$\n\x0ejudge_base_url\x18\x06 \x01(\tR\x0cjudgeBaseUrlB\x0c\n\n_threshold\"\xe0\x01\n\rCodeEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x12\n\x04path\x18\x06 \x01(\tR\x04pathB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xf8\x01\n\x0fRemoteEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x16\n\x06source\x18\x06 \x01(\tR\x06source\x12\x10\n\x03ref\x18\x07 \x01(\tR\x03refB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xdc\x01\n\tEvaluator\x12\x45\n\x07\x62uiltin\x18\x01 \x01(\x0b\x32).agentevals.api.v1alpha1.BuiltinEvaluatorH\x00R\x07\x62uiltin\x12<\n\x04\x63ode\x18\x02 \x01(\x0b\x32&.agentevals.api.v1alpha1.CodeEvaluatorH\x00R\x04\x63ode\x12\x42\n\x06remote\x18\x03 \x01(\x0b\x32(.agentevals.api.v1alpha1.RemoteEvaluatorH\x00R\x06remoteB\x06\n\x04kind\"\xb0\x02\n\nEvalParams\x12\x42\n\nevaluators\x18\x01 \x03(\x0b\x32\".agentevals.api.v1alpha1.EvaluatorR\nevaluators\x12\x37\n\x15max_concurrent_traces\x18\x02 \x01(\x05H\x00R\x13maxConcurrentTraces\x88\x01\x01\x12\x35\n\x14max_concurrent_evals\x18\x03 \x01(\x05H\x01R\x12maxConcurrentEvals\x88\x01\x01\x12;\n\x08group_by\x18\x04 \x01(\x0e\x32 .agentevals.api.v1alpha1.GroupByR\x07groupByB\x18\n\x16_max_concurrent_tracesB\x17\n\x15_max_concurrent_evals\"t\n\x10InvocationResult\x12\x19\n\x05score\x18\x01 \x01(\x01H\x00R\x05score\x88\x01\x01\x12;\n\x06status\x18\x02 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\x06statusB\x08\n\x06_score\"\xad\x03\n\x0cMetricResult\x12\x1f\n\x0bmetric_name\x18\x01 \x01(\tR\nmetricName\x12\x19\n\x05score\x18\x02 \x01(\x01H\x00R\x05score\x88\x01\x01\x12\x44\n\x0b\x65val_status\x18\x03 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\nevalStatus\x12P\n\x0eper_invocation\x18\x04 \x03(\x0b\x32).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n\x05\x65rror\x18\x05 \x01(\tH\x01R\x05\x65rror\x88\x01\x01\x12\"\n\nerror_type\x18\x06 \x01(\tH\x02R\terrorType\x88\x01\x01\x12\x31\n\x07\x64\x65tails\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x07\x64\x65tails\x12$\n\x0b\x64uration_ms\x18\x08 \x01(\x01H\x03R\ndurationMs\x88\x01\x01\x42\x08\n\x06_scoreB\x08\n\x06_errorB\r\n\x0b_error_typeB\x0e\n\x0c_duration_ms\"=\n\x07TurnRef\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x02 \x01(\tR\x06spanId\"\xc8\x04\n\x0bTraceResult\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\'\n\x0fnum_invocations\x18\x02 \x01(\x05R\x0enumInvocations\x12L\n\x0emetric_results\x18\x03 \x03(\x0b\x32%.agentevals.api.v1alpha1.MetricResultR\rmetricResults\x12/\n\x13\x63onversion_warnings\x18\x04 \x03(\tR\x12\x63onversionWarnings\x12H\n\x13performance_metrics\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12!\n\x0cservice_name\x18\x06 \x01(\tR\x0bserviceName\x12\x1d\n\nagent_name\x18\x07 \x01(\tR\tagentName\x12\x1b\n\ttrace_ids\x18\x08 \x03(\tR\x08traceIds\x12\'\n\x0f\x63onversation_id\x18\t \x01(\tR\x0e\x63onversationId\x12\x1b\n\tgroup_key\x18\n \x01(\tR\x08groupKey\x12 \n\x0c\x65val_case_id\x18\x0b \x01(\tR\nevalCaseId\x12&\n\x0f\x65val_case_match\x18\x0c \x01(\tR\revalCaseMatch\x12=\n\tturn_refs\x18\r \x03(\x0b\x32 .agentevals.api.v1alpha1.TurnRefR\x08turnRefs\"\xcf\x01\n\tRunResult\x12I\n\rtrace_results\x18\x01 \x03(\x0b\x32$.agentevals.api.v1alpha1.TraceResultR\x0ctraceResults\x12\x16\n\x06\x65rrors\x18\x02 \x03(\tR\x06\x65rrors\x12H\n\x13performance_metrics\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12\x15\n\x06run_id\x18\x04 \x01(\tR\x05runId\"\xa3\x03\n\x15\x45valuateTracesRequest\x12;\n\x06traces\x18\x01 \x01(\x0b\x32#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n\x0c\x61\x64k_eval_set\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\nadkEvalSet\x12;\n\x06params\x18\x03 \x01(\x0b\x32#.agentevals.api.v1alpha1.EvalParamsR\x06params\x12k\n\x0f\x63redential_refs\x18\x04 \x03(\x0b\x32\x42.agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntryR\x0e\x63redentialRefs\x1aZ\n\x13\x43redentialRefsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x05value:\x02\x38\x01\x42\n\n\x08\x65val_set\".\n\x12\x45valuationProgress\x12\x18\n\x07message\x18\x01 \x01(\tR\x07message\"\xb7\x01\n\x10GroupPerformance\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12H\n\x13performance_metrics\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12>\n\x0etrace_metadata\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\rtraceMetadata\"\xc0\x02\n\x16\x45valuateTracesResponse\x12I\n\x08progress\x18\x01 \x01(\x0b\x32+.agentevals.api.v1alpha1.EvaluationProgressH\x00R\x08progress\x12M\n\x0bperformance\x18\x02 \x01(\x0b\x32).agentevals.api.v1alpha1.GroupPerformanceH\x00R\x0bperformance\x12I\n\x0ctrace_result\x18\x03 \x01(\x0b\x32$.agentevals.api.v1alpha1.TraceResultH\x00R\x0btraceResult\x12\x38\n\x04\x64one\x18\x04 \x01(\x0b\x32\".agentevals.api.v1alpha1.RunResultH\x00R\x04\x64oneB\x07\n\x05\x65vent*\xa6\x01\n\x13TrajectoryMatchType\x12%\n!TRAJECTORY_MATCH_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bTRAJECTORY_MATCH_TYPE_EXACT\x10\x01\x12\"\n\x1eTRAJECTORY_MATCH_TYPE_IN_ORDER\x10\x02\x12#\n\x1fTRAJECTORY_MATCH_TYPE_ANY_ORDER\x10\x03*x\n\nEvalStatus\x12\x1b\n\x17\x45VAL_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12\x45VAL_STATUS_PASSED\x10\x01\x12\x16\n\x12\x45VAL_STATUS_FAILED\x10\x02\x12\x1d\n\x19\x45VAL_STATUS_NOT_EVALUATED\x10\x03\x32\xf2\x01\n\x11\x45valuationService\x12h\n\x0bListMetrics\x12+.agentevals.api.v1alpha1.ListMetricsRequest\x1a,.agentevals.api.v1alpha1.ListMetricsResponse\x12s\n\x0e\x45valuateTraces\x12..agentevals.api.v1alpha1.EvaluateTracesRequest\x1a/.agentevals.api.v1alpha1.EvaluateTracesResponse0\x01\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.evaluation_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._loaded_options = None + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_options = b'8\001' + _globals['_TRAJECTORYMATCHTYPE']._serialized_start=4213 + _globals['_TRAJECTORYMATCHTYPE']._serialized_end=4379 + _globals['_EVALSTATUS']._serialized_start=4381 + _globals['_EVALSTATUS']._serialized_end=4501 + _globals['_LISTMETRICSREQUEST']._serialized_start=137 + _globals['_LISTMETRICSREQUEST']._serialized_end=157 + _globals['_METRIC']._serialized_start=160 + _globals['_METRIC']._serialized_end=433 + _globals['_LISTMETRICSRESPONSE']._serialized_start=435 + _globals['_LISTMETRICSRESPONSE']._serialized_end=515 + _globals['_BUILTINEVALUATOR']._serialized_start=518 + _globals['_BUILTINEVALUATOR']._serialized_end=813 + _globals['_CODEEVALUATOR']._serialized_start=816 + _globals['_CODEEVALUATOR']._serialized_end=1040 + _globals['_REMOTEEVALUATOR']._serialized_start=1043 + _globals['_REMOTEEVALUATOR']._serialized_end=1291 + _globals['_EVALUATOR']._serialized_start=1294 + _globals['_EVALUATOR']._serialized_end=1514 + _globals['_EVALPARAMS']._serialized_start=1517 + _globals['_EVALPARAMS']._serialized_end=1821 + _globals['_INVOCATIONRESULT']._serialized_start=1823 + _globals['_INVOCATIONRESULT']._serialized_end=1939 + _globals['_METRICRESULT']._serialized_start=1942 + _globals['_METRICRESULT']._serialized_end=2371 + _globals['_TURNREF']._serialized_start=2373 + _globals['_TURNREF']._serialized_end=2434 + _globals['_TRACERESULT']._serialized_start=2437 + _globals['_TRACERESULT']._serialized_end=3021 + _globals['_RUNRESULT']._serialized_start=3024 + _globals['_RUNRESULT']._serialized_end=3231 + _globals['_EVALUATETRACESREQUEST']._serialized_start=3234 + _globals['_EVALUATETRACESREQUEST']._serialized_end=3653 + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_start=3551 + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_end=3641 + _globals['_EVALUATIONPROGRESS']._serialized_start=3655 + _globals['_EVALUATIONPROGRESS']._serialized_end=3701 + _globals['_GROUPPERFORMANCE']._serialized_start=3704 + _globals['_GROUPPERFORMANCE']._serialized_end=3887 + _globals['_EVALUATETRACESRESPONSE']._serialized_start=3890 + _globals['_EVALUATETRACESRESPONSE']._serialized_end=4210 + _globals['_EVALUATIONSERVICE']._serialized_start=4504 + _globals['_EVALUATIONSERVICE']._serialized_end=4746 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/evaluation_pb2.pyi b/src/agentevals/api/v1alpha1/evaluation_pb2.pyi new file mode 100644 index 00000000..03da133c --- /dev/null +++ b/src/agentevals/api/v1alpha1/evaluation_pb2.pyi @@ -0,0 +1,259 @@ +from agentevals.api.v1alpha1 import common_pb2 as _common_pb2 +from google.protobuf import struct_pb2 as _struct_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class TrajectoryMatchType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + TRAJECTORY_MATCH_TYPE_UNSPECIFIED: _ClassVar[TrajectoryMatchType] + TRAJECTORY_MATCH_TYPE_EXACT: _ClassVar[TrajectoryMatchType] + TRAJECTORY_MATCH_TYPE_IN_ORDER: _ClassVar[TrajectoryMatchType] + TRAJECTORY_MATCH_TYPE_ANY_ORDER: _ClassVar[TrajectoryMatchType] + +class EvalStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + EVAL_STATUS_UNSPECIFIED: _ClassVar[EvalStatus] + EVAL_STATUS_PASSED: _ClassVar[EvalStatus] + EVAL_STATUS_FAILED: _ClassVar[EvalStatus] + EVAL_STATUS_NOT_EVALUATED: _ClassVar[EvalStatus] +TRAJECTORY_MATCH_TYPE_UNSPECIFIED: TrajectoryMatchType +TRAJECTORY_MATCH_TYPE_EXACT: TrajectoryMatchType +TRAJECTORY_MATCH_TYPE_IN_ORDER: TrajectoryMatchType +TRAJECTORY_MATCH_TYPE_ANY_ORDER: TrajectoryMatchType +EVAL_STATUS_UNSPECIFIED: EvalStatus +EVAL_STATUS_PASSED: EvalStatus +EVAL_STATUS_FAILED: EvalStatus +EVAL_STATUS_NOT_EVALUATED: EvalStatus + +class ListMetricsRequest(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class Metric(_message.Message): + __slots__ = ("name", "category", "description", "requires_eval_set", "requires_llm", "requires_gcp", "requires_rubrics", "working") + NAME_FIELD_NUMBER: _ClassVar[int] + CATEGORY_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + REQUIRES_EVAL_SET_FIELD_NUMBER: _ClassVar[int] + REQUIRES_LLM_FIELD_NUMBER: _ClassVar[int] + REQUIRES_GCP_FIELD_NUMBER: _ClassVar[int] + REQUIRES_RUBRICS_FIELD_NUMBER: _ClassVar[int] + WORKING_FIELD_NUMBER: _ClassVar[int] + name: str + category: str + description: str + requires_eval_set: bool + requires_llm: bool + requires_gcp: bool + requires_rubrics: bool + working: bool + def __init__(self, name: _Optional[str] = ..., category: _Optional[str] = ..., description: _Optional[str] = ..., requires_eval_set: _Optional[bool] = ..., requires_llm: _Optional[bool] = ..., requires_gcp: _Optional[bool] = ..., requires_rubrics: _Optional[bool] = ..., working: _Optional[bool] = ...) -> None: ... + +class ListMetricsResponse(_message.Message): + __slots__ = ("metrics",) + METRICS_FIELD_NUMBER: _ClassVar[int] + metrics: _containers.RepeatedCompositeFieldContainer[Metric] + def __init__(self, metrics: _Optional[_Iterable[_Union[Metric, _Mapping]]] = ...) -> None: ... + +class BuiltinEvaluator(_message.Message): + __slots__ = ("name", "threshold", "judge_model", "trajectory_match_type", "credential_ref", "judge_base_url") + NAME_FIELD_NUMBER: _ClassVar[int] + THRESHOLD_FIELD_NUMBER: _ClassVar[int] + JUDGE_MODEL_FIELD_NUMBER: _ClassVar[int] + TRAJECTORY_MATCH_TYPE_FIELD_NUMBER: _ClassVar[int] + CREDENTIAL_REF_FIELD_NUMBER: _ClassVar[int] + JUDGE_BASE_URL_FIELD_NUMBER: _ClassVar[int] + name: str + threshold: float + judge_model: str + trajectory_match_type: TrajectoryMatchType + credential_ref: str + judge_base_url: str + def __init__(self, name: _Optional[str] = ..., threshold: _Optional[float] = ..., judge_model: _Optional[str] = ..., trajectory_match_type: _Optional[_Union[TrajectoryMatchType, str]] = ..., credential_ref: _Optional[str] = ..., judge_base_url: _Optional[str] = ...) -> None: ... + +class CodeEvaluator(_message.Message): + __slots__ = ("name", "threshold", "timeout", "config", "executor", "path") + NAME_FIELD_NUMBER: _ClassVar[int] + THRESHOLD_FIELD_NUMBER: _ClassVar[int] + TIMEOUT_FIELD_NUMBER: _ClassVar[int] + CONFIG_FIELD_NUMBER: _ClassVar[int] + EXECUTOR_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + name: str + threshold: float + timeout: int + config: _struct_pb2.Struct + executor: str + path: str + def __init__(self, name: _Optional[str] = ..., threshold: _Optional[float] = ..., timeout: _Optional[int] = ..., config: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., executor: _Optional[str] = ..., path: _Optional[str] = ...) -> None: ... + +class RemoteEvaluator(_message.Message): + __slots__ = ("name", "threshold", "timeout", "config", "executor", "source", "ref") + NAME_FIELD_NUMBER: _ClassVar[int] + THRESHOLD_FIELD_NUMBER: _ClassVar[int] + TIMEOUT_FIELD_NUMBER: _ClassVar[int] + CONFIG_FIELD_NUMBER: _ClassVar[int] + EXECUTOR_FIELD_NUMBER: _ClassVar[int] + SOURCE_FIELD_NUMBER: _ClassVar[int] + REF_FIELD_NUMBER: _ClassVar[int] + name: str + threshold: float + timeout: int + config: _struct_pb2.Struct + executor: str + source: str + ref: str + def __init__(self, name: _Optional[str] = ..., threshold: _Optional[float] = ..., timeout: _Optional[int] = ..., config: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., executor: _Optional[str] = ..., source: _Optional[str] = ..., ref: _Optional[str] = ...) -> None: ... + +class Evaluator(_message.Message): + __slots__ = ("builtin", "code", "remote") + BUILTIN_FIELD_NUMBER: _ClassVar[int] + CODE_FIELD_NUMBER: _ClassVar[int] + REMOTE_FIELD_NUMBER: _ClassVar[int] + builtin: BuiltinEvaluator + code: CodeEvaluator + remote: RemoteEvaluator + def __init__(self, builtin: _Optional[_Union[BuiltinEvaluator, _Mapping]] = ..., code: _Optional[_Union[CodeEvaluator, _Mapping]] = ..., remote: _Optional[_Union[RemoteEvaluator, _Mapping]] = ...) -> None: ... + +class EvalParams(_message.Message): + __slots__ = ("evaluators", "max_concurrent_traces", "max_concurrent_evals", "group_by") + EVALUATORS_FIELD_NUMBER: _ClassVar[int] + MAX_CONCURRENT_TRACES_FIELD_NUMBER: _ClassVar[int] + MAX_CONCURRENT_EVALS_FIELD_NUMBER: _ClassVar[int] + GROUP_BY_FIELD_NUMBER: _ClassVar[int] + evaluators: _containers.RepeatedCompositeFieldContainer[Evaluator] + max_concurrent_traces: int + max_concurrent_evals: int + group_by: _common_pb2.GroupBy + def __init__(self, evaluators: _Optional[_Iterable[_Union[Evaluator, _Mapping]]] = ..., max_concurrent_traces: _Optional[int] = ..., max_concurrent_evals: _Optional[int] = ..., group_by: _Optional[_Union[_common_pb2.GroupBy, str]] = ...) -> None: ... + +class InvocationResult(_message.Message): + __slots__ = ("score", "status") + SCORE_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + score: float + status: EvalStatus + def __init__(self, score: _Optional[float] = ..., status: _Optional[_Union[EvalStatus, str]] = ...) -> None: ... + +class MetricResult(_message.Message): + __slots__ = ("metric_name", "score", "eval_status", "per_invocation", "error", "error_type", "details", "duration_ms") + METRIC_NAME_FIELD_NUMBER: _ClassVar[int] + SCORE_FIELD_NUMBER: _ClassVar[int] + EVAL_STATUS_FIELD_NUMBER: _ClassVar[int] + PER_INVOCATION_FIELD_NUMBER: _ClassVar[int] + ERROR_FIELD_NUMBER: _ClassVar[int] + ERROR_TYPE_FIELD_NUMBER: _ClassVar[int] + DETAILS_FIELD_NUMBER: _ClassVar[int] + DURATION_MS_FIELD_NUMBER: _ClassVar[int] + metric_name: str + score: float + eval_status: EvalStatus + per_invocation: _containers.RepeatedCompositeFieldContainer[InvocationResult] + error: str + error_type: str + details: _struct_pb2.Struct + duration_ms: float + def __init__(self, metric_name: _Optional[str] = ..., score: _Optional[float] = ..., eval_status: _Optional[_Union[EvalStatus, str]] = ..., per_invocation: _Optional[_Iterable[_Union[InvocationResult, _Mapping]]] = ..., error: _Optional[str] = ..., error_type: _Optional[str] = ..., details: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., duration_ms: _Optional[float] = ...) -> None: ... + +class TurnRef(_message.Message): + __slots__ = ("trace_id", "span_id") + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + trace_id: str + span_id: str + def __init__(self, trace_id: _Optional[str] = ..., span_id: _Optional[str] = ...) -> None: ... + +class TraceResult(_message.Message): + __slots__ = ("trace_id", "num_invocations", "metric_results", "conversion_warnings", "performance_metrics", "service_name", "agent_name", "trace_ids", "conversation_id", "group_key", "eval_case_id", "eval_case_match", "turn_refs") + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + NUM_INVOCATIONS_FIELD_NUMBER: _ClassVar[int] + METRIC_RESULTS_FIELD_NUMBER: _ClassVar[int] + CONVERSION_WARNINGS_FIELD_NUMBER: _ClassVar[int] + PERFORMANCE_METRICS_FIELD_NUMBER: _ClassVar[int] + SERVICE_NAME_FIELD_NUMBER: _ClassVar[int] + AGENT_NAME_FIELD_NUMBER: _ClassVar[int] + TRACE_IDS_FIELD_NUMBER: _ClassVar[int] + CONVERSATION_ID_FIELD_NUMBER: _ClassVar[int] + GROUP_KEY_FIELD_NUMBER: _ClassVar[int] + EVAL_CASE_ID_FIELD_NUMBER: _ClassVar[int] + EVAL_CASE_MATCH_FIELD_NUMBER: _ClassVar[int] + TURN_REFS_FIELD_NUMBER: _ClassVar[int] + trace_id: str + num_invocations: int + metric_results: _containers.RepeatedCompositeFieldContainer[MetricResult] + conversion_warnings: _containers.RepeatedScalarFieldContainer[str] + performance_metrics: _struct_pb2.Struct + service_name: str + agent_name: str + trace_ids: _containers.RepeatedScalarFieldContainer[str] + conversation_id: str + group_key: str + eval_case_id: str + eval_case_match: str + turn_refs: _containers.RepeatedCompositeFieldContainer[TurnRef] + def __init__(self, trace_id: _Optional[str] = ..., num_invocations: _Optional[int] = ..., metric_results: _Optional[_Iterable[_Union[MetricResult, _Mapping]]] = ..., conversion_warnings: _Optional[_Iterable[str]] = ..., performance_metrics: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., service_name: _Optional[str] = ..., agent_name: _Optional[str] = ..., trace_ids: _Optional[_Iterable[str]] = ..., conversation_id: _Optional[str] = ..., group_key: _Optional[str] = ..., eval_case_id: _Optional[str] = ..., eval_case_match: _Optional[str] = ..., turn_refs: _Optional[_Iterable[_Union[TurnRef, _Mapping]]] = ...) -> None: ... + +class RunResult(_message.Message): + __slots__ = ("trace_results", "errors", "performance_metrics", "run_id") + TRACE_RESULTS_FIELD_NUMBER: _ClassVar[int] + ERRORS_FIELD_NUMBER: _ClassVar[int] + PERFORMANCE_METRICS_FIELD_NUMBER: _ClassVar[int] + RUN_ID_FIELD_NUMBER: _ClassVar[int] + trace_results: _containers.RepeatedCompositeFieldContainer[TraceResult] + errors: _containers.RepeatedScalarFieldContainer[str] + performance_metrics: _struct_pb2.Struct + run_id: str + def __init__(self, trace_results: _Optional[_Iterable[_Union[TraceResult, _Mapping]]] = ..., errors: _Optional[_Iterable[str]] = ..., performance_metrics: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., run_id: _Optional[str] = ...) -> None: ... + +class EvaluateTracesRequest(_message.Message): + __slots__ = ("traces", "adk_eval_set", "params", "credential_refs") + class CredentialRefsEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: _struct_pb2.Struct + def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + TRACES_FIELD_NUMBER: _ClassVar[int] + ADK_EVAL_SET_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] + CREDENTIAL_REFS_FIELD_NUMBER: _ClassVar[int] + traces: _common_pb2.TraceInput + adk_eval_set: _struct_pb2.Struct + params: EvalParams + credential_refs: _containers.MessageMap[str, _struct_pb2.Struct] + def __init__(self, traces: _Optional[_Union[_common_pb2.TraceInput, _Mapping]] = ..., adk_eval_set: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., params: _Optional[_Union[EvalParams, _Mapping]] = ..., credential_refs: _Optional[_Mapping[str, _struct_pb2.Struct]] = ...) -> None: ... + +class EvaluationProgress(_message.Message): + __slots__ = ("message",) + MESSAGE_FIELD_NUMBER: _ClassVar[int] + message: str + def __init__(self, message: _Optional[str] = ...) -> None: ... + +class GroupPerformance(_message.Message): + __slots__ = ("trace_id", "performance_metrics", "trace_metadata") + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + PERFORMANCE_METRICS_FIELD_NUMBER: _ClassVar[int] + TRACE_METADATA_FIELD_NUMBER: _ClassVar[int] + trace_id: str + performance_metrics: _struct_pb2.Struct + trace_metadata: _struct_pb2.Struct + def __init__(self, trace_id: _Optional[str] = ..., performance_metrics: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., trace_metadata: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + +class EvaluateTracesResponse(_message.Message): + __slots__ = ("progress", "performance", "trace_result", "done") + PROGRESS_FIELD_NUMBER: _ClassVar[int] + PERFORMANCE_FIELD_NUMBER: _ClassVar[int] + TRACE_RESULT_FIELD_NUMBER: _ClassVar[int] + DONE_FIELD_NUMBER: _ClassVar[int] + progress: EvaluationProgress + performance: GroupPerformance + trace_result: TraceResult + done: RunResult + def __init__(self, progress: _Optional[_Union[EvaluationProgress, _Mapping]] = ..., performance: _Optional[_Union[GroupPerformance, _Mapping]] = ..., trace_result: _Optional[_Union[TraceResult, _Mapping]] = ..., done: _Optional[_Union[RunResult, _Mapping]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/evaluation_pb2_grpc.py b/src/agentevals/api/v1alpha1/evaluation_pb2_grpc.py new file mode 100644 index 00000000..64e983ae --- /dev/null +++ b/src/agentevals/api/v1alpha1/evaluation_pb2_grpc.py @@ -0,0 +1,123 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agentevals.api.v1alpha1 import evaluation_pb2 as agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2 + + +class EvaluationServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListMetrics = channel.unary_unary( + '/agentevals.api.v1alpha1.EvaluationService/ListMetrics', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsResponse.FromString, + _registered_method=True) + self.EvaluateTraces = channel.unary_stream( + '/agentevals.api.v1alpha1.EvaluationService/EvaluateTraces', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesResponse.FromString, + _registered_method=True) + + +class EvaluationServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ListMetrics(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def EvaluateTraces(self, request, context): + """Evaluates traces. Streams progress, each group's performance metrics before evaluation, each + group's results, and the final result, after which the stream ends. When run storage is + configured, the evaluation is also recorded as a run. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_EvaluationServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListMetrics': grpc.unary_unary_rpc_method_handler( + servicer.ListMetrics, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsResponse.SerializeToString, + ), + 'EvaluateTraces': grpc.unary_stream_rpc_method_handler( + servicer.EvaluateTraces, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agentevals.api.v1alpha1.EvaluationService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agentevals.api.v1alpha1.EvaluationService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class EvaluationService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ListMetrics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.EvaluationService/ListMetrics', + agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.ListMetricsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def EvaluateTraces(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/agentevals.api.v1alpha1.EvaluationService/EvaluateTraces', + agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2.EvaluateTracesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/agentevals/api/v1alpha1/runs_pb2.py b/src/agentevals/api/v1alpha1/runs_pb2.py new file mode 100644 index 00000000..4332cd50 --- /dev/null +++ b/src/agentevals/api/v1alpha1/runs_pb2.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/runs.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/runs.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agentevals.api.v1alpha1 import common_pb2 as agentevals_dot_api_dot_v1alpha1_dot_common__pb2 +from agentevals.api.v1alpha1 import evaluation_pb2 as agentevals_dot_api_dot_v1alpha1_dot_evaluation__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\"agentevals/api/v1alpha1/runs.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a(agentevals/api/v1alpha1/evaluation.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x85\x01\n\x0fHttpTraceTarget\x12\x19\n\x08\x62\x61se_url\x18\x01 \x01(\tR\x07\x62\x61seUrl\x12\x19\n\x08trace_id\x18\x02 \x01(\tR\x07traceId\x12<\n\x06\x66ormat\x18\x03 \x01(\x0e\x32$.agentevals.api.v1alpha1.TraceFormatR\x06\x66ormat\"R\n\x0eUploadedTraces\x12\x1f\n\x0btrace_count\x18\x01 \x01(\x05R\ntraceCount\x12\x1f\n\x0btrace_files\x18\x02 \x03(\tR\ntraceFiles\"\x9c\x02\n\x0bTraceTarget\x12@\n\x06inline\x18\x01 \x01(\x0b\x32&.agentevals.api.v1alpha1.TraceDocumentH\x00R\x06inline\x12<\n\x04otlp\x18\x02 \x01(\x0b\x32&.agentevals.api.v1alpha1.OtlpTelemetryH\x00R\x04otlp\x12>\n\x04http\x18\x03 \x01(\x0b\x32(.agentevals.api.v1alpha1.HttpTraceTargetH\x00R\x04http\x12\x45\n\x08uploaded\x18\x04 \x01(\x0b\x32\'.agentevals.api.v1alpha1.UploadedTracesH\x00R\x08uploadedB\x06\n\x04kind\"\x94\x01\n\nRunContext\x12J\n\x07headers\x18\x01 \x03(\x0b\x32\x30.agentevals.api.v1alpha1.RunContext.HeadersEntryR\x07headers\x1a:\n\x0cHeadersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\xc1\x04\n\x07RunSpec\x12@\n\x08\x61pproach\x18\x01 \x01(\x0e\x32$.agentevals.api.v1alpha1.RunApproachR\x08\x61pproach\x12<\n\x06target\x18\x02 \x01(\x0b\x32$.agentevals.api.v1alpha1.TraceTargetR\x06target\x12;\n\x0c\x61\x64k_eval_set\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\nadkEvalSet\x12\x44\n\x0b\x65val_config\x18\x04 \x01(\x0b\x32#.agentevals.api.v1alpha1.EvalParamsR\nevalConfig\x12-\n\x05sinks\x18\x05 \x03(\x0b\x32\x17.google.protobuf.StructR\x05sinks\x12=\n\x07\x63ontext\x18\x06 \x01(\x0b\x32#.agentevals.api.v1alpha1.RunContextR\x07\x63ontext\x12]\n\x0f\x63redential_refs\x18\x07 \x03(\x0b\x32\x34.agentevals.api.v1alpha1.RunSpec.CredentialRefsEntryR\x0e\x63redentialRefs\x1aZ\n\x13\x43redentialRefsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x05value:\x02\x38\x01\x42\n\n\x08\x65val_set\"\xec\x03\n\x03Run\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\x12:\n\x06status\x18\x02 \x01(\x0e\x32\".agentevals.api.v1alpha1.RunStatusR\x06status\x12\x34\n\x04spec\x18\x03 \x01(\x0b\x32 .agentevals.api.v1alpha1.RunSpecR\x04spec\x12\x18\n\x07\x61ttempt\x18\x04 \x01(\x05R\x07\x61ttempt\x12\x1b\n\tworker_id\x18\x05 \x01(\tR\x08workerId\x12\x14\n\x05\x65rror\x18\x06 \x01(\tR\x05\x65rror\x12\x31\n\x07summary\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x07summary\x12\x39\n\ncreated_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x39\n\nstarted_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12;\n\x0b\x66inished_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nfinishedAt\x12)\n\x10\x63\x61ncel_requested\x18\x0b \x01(\x08R\x0f\x63\x61ncelRequested\"\xec\x05\n\x06Result\x12\x1b\n\tresult_id\x18\x01 \x01(\tR\x08resultId\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId\x12\'\n\x10\x65val_set_item_id\x18\x03 \x01(\tR\revalSetItemId\x12+\n\x12\x65val_set_item_name\x18\x04 \x01(\tR\x0f\x65valSetItemName\x12%\n\x0e\x65valuator_name\x18\x05 \x01(\tR\revaluatorName\x12M\n\x0e\x65valuator_type\x18\x06 \x01(\x0e\x32&.agentevals.api.v1alpha1.EvaluatorTypeR\revaluatorType\x12=\n\x06status\x18\x07 \x01(\x0e\x32%.agentevals.api.v1alpha1.ResultStatusR\x06status\x12\x19\n\x05score\x18\x08 \x01(\x01H\x00R\x05score\x88\x01\x01\x12P\n\x0eper_invocation\x18\t \x03(\x0b\x32).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n\x08trace_id\x18\n \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x0b \x01(\tR\x06spanId\x12\x31\n\x07\x64\x65tails\x18\x0c \x01(\x0b\x32\x17.google.protobuf.StructR\x07\x64\x65tails\x12\x1d\n\nerror_text\x18\r \x01(\tR\terrorText\x12\x38\n\x0btokens_used\x18\x0e \x01(\x0b\x32\x17.google.protobuf.StructR\ntokensUsed\x12\"\n\nlatency_ms\x18\x0f \x01(\x03H\x01R\tlatencyMs\x88\x01\x01\x12\x39\n\ncreated_at\x18\x10 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAtB\x08\n\x06_scoreB\r\n\x0b_latency_ms\"_\n\x10SubmitRunRequest\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x34\n\x04spec\x18\x02 \x01(\x0b\x32 .agentevals.api.v1alpha1.RunSpecR\x04spec\"C\n\x11SubmitRunResponse\x12.\n\x03run\x18\x01 \x01(\x0b\x32\x1c.agentevals.api.v1alpha1.RunR\x03run\"&\n\rGetRunRequest\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\"@\n\x0eGetRunResponse\x12.\n\x03run\x18\x01 \x01(\x0b\x32\x1c.agentevals.api.v1alpha1.RunR\x03run\"\x8b\x01\n\x0fListRunsRequest\x12>\n\x08statuses\x18\x01 \x03(\x0e\x32\".agentevals.api.v1alpha1.RunStatusR\x08statuses\x12\x38\n\x04page\x18\x02 \x01(\x0b\x32$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x7f\n\x10ListRunsResponse\x12\x30\n\x04runs\x18\x01 \x03(\x0b\x32\x1c.agentevals.api.v1alpha1.RunR\x04runs\x12\x39\n\x04page\x18\x02 \x01(\x0b\x32%.agentevals.api.v1alpha1.PageResponseR\x04page\")\n\x10\x43\x61ncelRunRequest\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\"C\n\x11\x43\x61ncelRunResponse\x12.\n\x03run\x18\x01 \x01(\x0b\x32\x1c.agentevals.api.v1alpha1.RunR\x03run\"h\n\x15ListRunResultsRequest\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x38\n\x04page\x18\x02 \x01(\x0b\x32$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x8e\x01\n\x16ListRunResultsResponse\x12\x39\n\x07results\x18\x01 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.ResultR\x07results\x12\x39\n\x04page\x18\x02 \x01(\x0b\x32%.agentevals.api.v1alpha1.PageResponseR\x04page\"(\n\x0fWatchRunRequest\x12\x15\n\x06run_id\x18\x01 \x01(\tR\x05runId\"}\n\x10WatchRunResponse\x12.\n\x03run\x18\x01 \x01(\x0b\x32\x1c.agentevals.api.v1alpha1.RunR\x03run\x12\x39\n\x07results\x18\x02 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.ResultR\x07results*\xa1\x01\n\tRunStatus\x12\x1a\n\x16RUN_STATUS_UNSPECIFIED\x10\x00\x12\x15\n\x11RUN_STATUS_QUEUED\x10\x01\x12\x16\n\x12RUN_STATUS_RUNNING\x10\x02\x12\x18\n\x14RUN_STATUS_SUCCEEDED\x10\x03\x12\x15\n\x11RUN_STATUS_FAILED\x10\x04\x12\x18\n\x14RUN_STATUS_CANCELLED\x10\x05*\x97\x01\n\x0cResultStatus\x12\x1d\n\x19RESULT_STATUS_UNSPECIFIED\x10\x00\x12\x18\n\x14RESULT_STATUS_PASSED\x10\x01\x12\x18\n\x14RESULT_STATUS_FAILED\x10\x02\x12\x19\n\x15RESULT_STATUS_ERRORED\x10\x03\x12\x19\n\x15RESULT_STATUS_SKIPPED\x10\x04*\x7f\n\rEvaluatorType\x12\x1e\n\x1a\x45VALUATOR_TYPE_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x45VALUATOR_TYPE_BUILTIN\x10\x01\x12\x17\n\x13\x45VALUATOR_TYPE_CODE\x10\x02\x12\x19\n\x15\x45VALUATOR_TYPE_REMOTE\x10\x03*J\n\x0bRunApproach\x12\x1c\n\x18RUN_APPROACH_UNSPECIFIED\x10\x00\x12\x1d\n\x19RUN_APPROACH_TRACE_REPLAY\x10\x01\x32\xe6\x04\n\nRunService\x12\x62\n\tSubmitRun\x12).agentevals.api.v1alpha1.SubmitRunRequest\x1a*.agentevals.api.v1alpha1.SubmitRunResponse\x12Y\n\x06GetRun\x12&.agentevals.api.v1alpha1.GetRunRequest\x1a\'.agentevals.api.v1alpha1.GetRunResponse\x12_\n\x08ListRuns\x12(.agentevals.api.v1alpha1.ListRunsRequest\x1a).agentevals.api.v1alpha1.ListRunsResponse\x12\x62\n\tCancelRun\x12).agentevals.api.v1alpha1.CancelRunRequest\x1a*.agentevals.api.v1alpha1.CancelRunResponse\x12q\n\x0eListRunResults\x12..agentevals.api.v1alpha1.ListRunResultsRequest\x1a/.agentevals.api.v1alpha1.ListRunResultsResponse\x12\x61\n\x08WatchRun\x12(.agentevals.api.v1alpha1.WatchRunRequest\x1a).agentevals.api.v1alpha1.WatchRunResponse0\x01\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.runs_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_RUNCONTEXT_HEADERSENTRY']._loaded_options = None + _globals['_RUNCONTEXT_HEADERSENTRY']._serialized_options = b'8\001' + _globals['_RUNSPEC_CREDENTIALREFSENTRY']._loaded_options = None + _globals['_RUNSPEC_CREDENTIALREFSENTRY']._serialized_options = b'8\001' + _globals['_RUNSTATUS']._serialized_start=3766 + _globals['_RUNSTATUS']._serialized_end=3927 + _globals['_RESULTSTATUS']._serialized_start=3930 + _globals['_RESULTSTATUS']._serialized_end=4081 + _globals['_EVALUATORTYPE']._serialized_start=4083 + _globals['_EVALUATORTYPE']._serialized_end=4210 + _globals['_RUNAPPROACH']._serialized_start=4212 + _globals['_RUNAPPROACH']._serialized_end=4286 + _globals['_HTTPTRACETARGET']._serialized_start=207 + _globals['_HTTPTRACETARGET']._serialized_end=340 + _globals['_UPLOADEDTRACES']._serialized_start=342 + _globals['_UPLOADEDTRACES']._serialized_end=424 + _globals['_TRACETARGET']._serialized_start=427 + _globals['_TRACETARGET']._serialized_end=711 + _globals['_RUNCONTEXT']._serialized_start=714 + _globals['_RUNCONTEXT']._serialized_end=862 + _globals['_RUNCONTEXT_HEADERSENTRY']._serialized_start=804 + _globals['_RUNCONTEXT_HEADERSENTRY']._serialized_end=862 + _globals['_RUNSPEC']._serialized_start=865 + _globals['_RUNSPEC']._serialized_end=1442 + _globals['_RUNSPEC_CREDENTIALREFSENTRY']._serialized_start=1340 + _globals['_RUNSPEC_CREDENTIALREFSENTRY']._serialized_end=1430 + _globals['_RUN']._serialized_start=1445 + _globals['_RUN']._serialized_end=1937 + _globals['_RESULT']._serialized_start=1940 + _globals['_RESULT']._serialized_end=2688 + _globals['_SUBMITRUNREQUEST']._serialized_start=2690 + _globals['_SUBMITRUNREQUEST']._serialized_end=2785 + _globals['_SUBMITRUNRESPONSE']._serialized_start=2787 + _globals['_SUBMITRUNRESPONSE']._serialized_end=2854 + _globals['_GETRUNREQUEST']._serialized_start=2856 + _globals['_GETRUNREQUEST']._serialized_end=2894 + _globals['_GETRUNRESPONSE']._serialized_start=2896 + _globals['_GETRUNRESPONSE']._serialized_end=2960 + _globals['_LISTRUNSREQUEST']._serialized_start=2963 + _globals['_LISTRUNSREQUEST']._serialized_end=3102 + _globals['_LISTRUNSRESPONSE']._serialized_start=3104 + _globals['_LISTRUNSRESPONSE']._serialized_end=3231 + _globals['_CANCELRUNREQUEST']._serialized_start=3233 + _globals['_CANCELRUNREQUEST']._serialized_end=3274 + _globals['_CANCELRUNRESPONSE']._serialized_start=3276 + _globals['_CANCELRUNRESPONSE']._serialized_end=3343 + _globals['_LISTRUNRESULTSREQUEST']._serialized_start=3345 + _globals['_LISTRUNRESULTSREQUEST']._serialized_end=3449 + _globals['_LISTRUNRESULTSRESPONSE']._serialized_start=3452 + _globals['_LISTRUNRESULTSRESPONSE']._serialized_end=3594 + _globals['_WATCHRUNREQUEST']._serialized_start=3596 + _globals['_WATCHRUNREQUEST']._serialized_end=3636 + _globals['_WATCHRUNRESPONSE']._serialized_start=3638 + _globals['_WATCHRUNRESPONSE']._serialized_end=3763 + _globals['_RUNSERVICE']._serialized_start=4289 + _globals['_RUNSERVICE']._serialized_end=4903 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/runs_pb2.pyi b/src/agentevals/api/v1alpha1/runs_pb2.pyi new file mode 100644 index 00000000..4adaa718 --- /dev/null +++ b/src/agentevals/api/v1alpha1/runs_pb2.pyi @@ -0,0 +1,274 @@ +import datetime + +from agentevals.api.v1alpha1 import common_pb2 as _common_pb2 +from agentevals.api.v1alpha1 import evaluation_pb2 as _evaluation_pb2 +from google.protobuf import struct_pb2 as _struct_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class RunStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + RUN_STATUS_UNSPECIFIED: _ClassVar[RunStatus] + RUN_STATUS_QUEUED: _ClassVar[RunStatus] + RUN_STATUS_RUNNING: _ClassVar[RunStatus] + RUN_STATUS_SUCCEEDED: _ClassVar[RunStatus] + RUN_STATUS_FAILED: _ClassVar[RunStatus] + RUN_STATUS_CANCELLED: _ClassVar[RunStatus] + +class ResultStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + RESULT_STATUS_UNSPECIFIED: _ClassVar[ResultStatus] + RESULT_STATUS_PASSED: _ClassVar[ResultStatus] + RESULT_STATUS_FAILED: _ClassVar[ResultStatus] + RESULT_STATUS_ERRORED: _ClassVar[ResultStatus] + RESULT_STATUS_SKIPPED: _ClassVar[ResultStatus] + +class EvaluatorType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + EVALUATOR_TYPE_UNSPECIFIED: _ClassVar[EvaluatorType] + EVALUATOR_TYPE_BUILTIN: _ClassVar[EvaluatorType] + EVALUATOR_TYPE_CODE: _ClassVar[EvaluatorType] + EVALUATOR_TYPE_REMOTE: _ClassVar[EvaluatorType] + +class RunApproach(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + RUN_APPROACH_UNSPECIFIED: _ClassVar[RunApproach] + RUN_APPROACH_TRACE_REPLAY: _ClassVar[RunApproach] +RUN_STATUS_UNSPECIFIED: RunStatus +RUN_STATUS_QUEUED: RunStatus +RUN_STATUS_RUNNING: RunStatus +RUN_STATUS_SUCCEEDED: RunStatus +RUN_STATUS_FAILED: RunStatus +RUN_STATUS_CANCELLED: RunStatus +RESULT_STATUS_UNSPECIFIED: ResultStatus +RESULT_STATUS_PASSED: ResultStatus +RESULT_STATUS_FAILED: ResultStatus +RESULT_STATUS_ERRORED: ResultStatus +RESULT_STATUS_SKIPPED: ResultStatus +EVALUATOR_TYPE_UNSPECIFIED: EvaluatorType +EVALUATOR_TYPE_BUILTIN: EvaluatorType +EVALUATOR_TYPE_CODE: EvaluatorType +EVALUATOR_TYPE_REMOTE: EvaluatorType +RUN_APPROACH_UNSPECIFIED: RunApproach +RUN_APPROACH_TRACE_REPLAY: RunApproach + +class HttpTraceTarget(_message.Message): + __slots__ = ("base_url", "trace_id", "format") + BASE_URL_FIELD_NUMBER: _ClassVar[int] + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + FORMAT_FIELD_NUMBER: _ClassVar[int] + base_url: str + trace_id: str + format: _common_pb2.TraceFormat + def __init__(self, base_url: _Optional[str] = ..., trace_id: _Optional[str] = ..., format: _Optional[_Union[_common_pb2.TraceFormat, str]] = ...) -> None: ... + +class UploadedTraces(_message.Message): + __slots__ = ("trace_count", "trace_files") + TRACE_COUNT_FIELD_NUMBER: _ClassVar[int] + TRACE_FILES_FIELD_NUMBER: _ClassVar[int] + trace_count: int + trace_files: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, trace_count: _Optional[int] = ..., trace_files: _Optional[_Iterable[str]] = ...) -> None: ... + +class TraceTarget(_message.Message): + __slots__ = ("inline", "otlp", "http", "uploaded") + INLINE_FIELD_NUMBER: _ClassVar[int] + OTLP_FIELD_NUMBER: _ClassVar[int] + HTTP_FIELD_NUMBER: _ClassVar[int] + UPLOADED_FIELD_NUMBER: _ClassVar[int] + inline: _common_pb2.TraceDocument + otlp: _common_pb2.OtlpTelemetry + http: HttpTraceTarget + uploaded: UploadedTraces + def __init__(self, inline: _Optional[_Union[_common_pb2.TraceDocument, _Mapping]] = ..., otlp: _Optional[_Union[_common_pb2.OtlpTelemetry, _Mapping]] = ..., http: _Optional[_Union[HttpTraceTarget, _Mapping]] = ..., uploaded: _Optional[_Union[UploadedTraces, _Mapping]] = ...) -> None: ... + +class RunContext(_message.Message): + __slots__ = ("headers",) + class HeadersEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + HEADERS_FIELD_NUMBER: _ClassVar[int] + headers: _containers.ScalarMap[str, str] + def __init__(self, headers: _Optional[_Mapping[str, str]] = ...) -> None: ... + +class RunSpec(_message.Message): + __slots__ = ("approach", "target", "adk_eval_set", "eval_config", "sinks", "context", "credential_refs") + class CredentialRefsEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: _struct_pb2.Struct + def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + APPROACH_FIELD_NUMBER: _ClassVar[int] + TARGET_FIELD_NUMBER: _ClassVar[int] + ADK_EVAL_SET_FIELD_NUMBER: _ClassVar[int] + EVAL_CONFIG_FIELD_NUMBER: _ClassVar[int] + SINKS_FIELD_NUMBER: _ClassVar[int] + CONTEXT_FIELD_NUMBER: _ClassVar[int] + CREDENTIAL_REFS_FIELD_NUMBER: _ClassVar[int] + approach: RunApproach + target: TraceTarget + adk_eval_set: _struct_pb2.Struct + eval_config: _evaluation_pb2.EvalParams + sinks: _containers.RepeatedCompositeFieldContainer[_struct_pb2.Struct] + context: RunContext + credential_refs: _containers.MessageMap[str, _struct_pb2.Struct] + def __init__(self, approach: _Optional[_Union[RunApproach, str]] = ..., target: _Optional[_Union[TraceTarget, _Mapping]] = ..., adk_eval_set: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., eval_config: _Optional[_Union[_evaluation_pb2.EvalParams, _Mapping]] = ..., sinks: _Optional[_Iterable[_Union[_struct_pb2.Struct, _Mapping]]] = ..., context: _Optional[_Union[RunContext, _Mapping]] = ..., credential_refs: _Optional[_Mapping[str, _struct_pb2.Struct]] = ...) -> None: ... + +class Run(_message.Message): + __slots__ = ("run_id", "status", "spec", "attempt", "worker_id", "error", "summary", "created_at", "started_at", "finished_at", "cancel_requested") + RUN_ID_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + SPEC_FIELD_NUMBER: _ClassVar[int] + ATTEMPT_FIELD_NUMBER: _ClassVar[int] + WORKER_ID_FIELD_NUMBER: _ClassVar[int] + ERROR_FIELD_NUMBER: _ClassVar[int] + SUMMARY_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + STARTED_AT_FIELD_NUMBER: _ClassVar[int] + FINISHED_AT_FIELD_NUMBER: _ClassVar[int] + CANCEL_REQUESTED_FIELD_NUMBER: _ClassVar[int] + run_id: str + status: RunStatus + spec: RunSpec + attempt: int + worker_id: str + error: str + summary: _struct_pb2.Struct + created_at: _timestamp_pb2.Timestamp + started_at: _timestamp_pb2.Timestamp + finished_at: _timestamp_pb2.Timestamp + cancel_requested: bool + def __init__(self, run_id: _Optional[str] = ..., status: _Optional[_Union[RunStatus, str]] = ..., spec: _Optional[_Union[RunSpec, _Mapping]] = ..., attempt: _Optional[int] = ..., worker_id: _Optional[str] = ..., error: _Optional[str] = ..., summary: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., finished_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., cancel_requested: _Optional[bool] = ...) -> None: ... + +class Result(_message.Message): + __slots__ = ("result_id", "run_id", "eval_set_item_id", "eval_set_item_name", "evaluator_name", "evaluator_type", "status", "score", "per_invocation", "trace_id", "span_id", "details", "error_text", "tokens_used", "latency_ms", "created_at") + RESULT_ID_FIELD_NUMBER: _ClassVar[int] + RUN_ID_FIELD_NUMBER: _ClassVar[int] + EVAL_SET_ITEM_ID_FIELD_NUMBER: _ClassVar[int] + EVAL_SET_ITEM_NAME_FIELD_NUMBER: _ClassVar[int] + EVALUATOR_NAME_FIELD_NUMBER: _ClassVar[int] + EVALUATOR_TYPE_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + SCORE_FIELD_NUMBER: _ClassVar[int] + PER_INVOCATION_FIELD_NUMBER: _ClassVar[int] + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + DETAILS_FIELD_NUMBER: _ClassVar[int] + ERROR_TEXT_FIELD_NUMBER: _ClassVar[int] + TOKENS_USED_FIELD_NUMBER: _ClassVar[int] + LATENCY_MS_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + result_id: str + run_id: str + eval_set_item_id: str + eval_set_item_name: str + evaluator_name: str + evaluator_type: EvaluatorType + status: ResultStatus + score: float + per_invocation: _containers.RepeatedCompositeFieldContainer[_evaluation_pb2.InvocationResult] + trace_id: str + span_id: str + details: _struct_pb2.Struct + error_text: str + tokens_used: _struct_pb2.Struct + latency_ms: int + created_at: _timestamp_pb2.Timestamp + def __init__(self, result_id: _Optional[str] = ..., run_id: _Optional[str] = ..., eval_set_item_id: _Optional[str] = ..., eval_set_item_name: _Optional[str] = ..., evaluator_name: _Optional[str] = ..., evaluator_type: _Optional[_Union[EvaluatorType, str]] = ..., status: _Optional[_Union[ResultStatus, str]] = ..., score: _Optional[float] = ..., per_invocation: _Optional[_Iterable[_Union[_evaluation_pb2.InvocationResult, _Mapping]]] = ..., trace_id: _Optional[str] = ..., span_id: _Optional[str] = ..., details: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., error_text: _Optional[str] = ..., tokens_used: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., latency_ms: _Optional[int] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... + +class SubmitRunRequest(_message.Message): + __slots__ = ("run_id", "spec") + RUN_ID_FIELD_NUMBER: _ClassVar[int] + SPEC_FIELD_NUMBER: _ClassVar[int] + run_id: str + spec: RunSpec + def __init__(self, run_id: _Optional[str] = ..., spec: _Optional[_Union[RunSpec, _Mapping]] = ...) -> None: ... + +class SubmitRunResponse(_message.Message): + __slots__ = ("run",) + RUN_FIELD_NUMBER: _ClassVar[int] + run: Run + def __init__(self, run: _Optional[_Union[Run, _Mapping]] = ...) -> None: ... + +class GetRunRequest(_message.Message): + __slots__ = ("run_id",) + RUN_ID_FIELD_NUMBER: _ClassVar[int] + run_id: str + def __init__(self, run_id: _Optional[str] = ...) -> None: ... + +class GetRunResponse(_message.Message): + __slots__ = ("run",) + RUN_FIELD_NUMBER: _ClassVar[int] + run: Run + def __init__(self, run: _Optional[_Union[Run, _Mapping]] = ...) -> None: ... + +class ListRunsRequest(_message.Message): + __slots__ = ("statuses", "page") + STATUSES_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + statuses: _containers.RepeatedScalarFieldContainer[RunStatus] + page: _common_pb2.PageRequest + def __init__(self, statuses: _Optional[_Iterable[_Union[RunStatus, str]]] = ..., page: _Optional[_Union[_common_pb2.PageRequest, _Mapping]] = ...) -> None: ... + +class ListRunsResponse(_message.Message): + __slots__ = ("runs", "page") + RUNS_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + runs: _containers.RepeatedCompositeFieldContainer[Run] + page: _common_pb2.PageResponse + def __init__(self, runs: _Optional[_Iterable[_Union[Run, _Mapping]]] = ..., page: _Optional[_Union[_common_pb2.PageResponse, _Mapping]] = ...) -> None: ... + +class CancelRunRequest(_message.Message): + __slots__ = ("run_id",) + RUN_ID_FIELD_NUMBER: _ClassVar[int] + run_id: str + def __init__(self, run_id: _Optional[str] = ...) -> None: ... + +class CancelRunResponse(_message.Message): + __slots__ = ("run",) + RUN_FIELD_NUMBER: _ClassVar[int] + run: Run + def __init__(self, run: _Optional[_Union[Run, _Mapping]] = ...) -> None: ... + +class ListRunResultsRequest(_message.Message): + __slots__ = ("run_id", "page") + RUN_ID_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + run_id: str + page: _common_pb2.PageRequest + def __init__(self, run_id: _Optional[str] = ..., page: _Optional[_Union[_common_pb2.PageRequest, _Mapping]] = ...) -> None: ... + +class ListRunResultsResponse(_message.Message): + __slots__ = ("results", "page") + RESULTS_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + results: _containers.RepeatedCompositeFieldContainer[Result] + page: _common_pb2.PageResponse + def __init__(self, results: _Optional[_Iterable[_Union[Result, _Mapping]]] = ..., page: _Optional[_Union[_common_pb2.PageResponse, _Mapping]] = ...) -> None: ... + +class WatchRunRequest(_message.Message): + __slots__ = ("run_id",) + RUN_ID_FIELD_NUMBER: _ClassVar[int] + run_id: str + def __init__(self, run_id: _Optional[str] = ...) -> None: ... + +class WatchRunResponse(_message.Message): + __slots__ = ("run", "results") + RUN_FIELD_NUMBER: _ClassVar[int] + RESULTS_FIELD_NUMBER: _ClassVar[int] + run: Run + results: _containers.RepeatedCompositeFieldContainer[Result] + def __init__(self, run: _Optional[_Union[Run, _Mapping]] = ..., results: _Optional[_Iterable[_Union[Result, _Mapping]]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/runs_pb2_grpc.py b/src/agentevals/api/v1alpha1/runs_pb2_grpc.py new file mode 100644 index 00000000..2ff4c147 --- /dev/null +++ b/src/agentevals/api/v1alpha1/runs_pb2_grpc.py @@ -0,0 +1,303 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agentevals.api.v1alpha1 import runs_pb2 as agentevals_dot_api_dot_v1alpha1_dot_runs__pb2 + + +class RunServiceStub(object): + """Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every + method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.SubmitRun = channel.unary_unary( + '/agentevals.api.v1alpha1.RunService/SubmitRun', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunResponse.FromString, + _registered_method=True) + self.GetRun = channel.unary_unary( + '/agentevals.api.v1alpha1.RunService/GetRun', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunResponse.FromString, + _registered_method=True) + self.ListRuns = channel.unary_unary( + '/agentevals.api.v1alpha1.RunService/ListRuns', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsResponse.FromString, + _registered_method=True) + self.CancelRun = channel.unary_unary( + '/agentevals.api.v1alpha1.RunService/CancelRun', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunResponse.FromString, + _registered_method=True) + self.ListRunResults = channel.unary_unary( + '/agentevals.api.v1alpha1.RunService/ListRunResults', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsResponse.FromString, + _registered_method=True) + self.WatchRun = channel.unary_stream( + '/agentevals.api.v1alpha1.RunService/WatchRun', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunResponse.FromString, + _registered_method=True) + + +class RunServiceServicer(object): + """Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every + method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. + """ + + def SubmitRun(self, request, context): + """Queues a run. Idempotent on run_id: the same id with the same spec returns the existing run; + a different spec fails with ALREADY_EXISTS and reason RUN_SPEC_CONFLICT. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetRun(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListRuns(self, request, context): + """Newest first. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CancelRun(self, request, context): + """Requests cancellation. Cancelling a finished run changes nothing. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListRunResults(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def WatchRun(self, request, context): + """The run and its results, then each change, until the run reaches a terminal status. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_RunServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'SubmitRun': grpc.unary_unary_rpc_method_handler( + servicer.SubmitRun, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunResponse.SerializeToString, + ), + 'GetRun': grpc.unary_unary_rpc_method_handler( + servicer.GetRun, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunResponse.SerializeToString, + ), + 'ListRuns': grpc.unary_unary_rpc_method_handler( + servicer.ListRuns, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsResponse.SerializeToString, + ), + 'CancelRun': grpc.unary_unary_rpc_method_handler( + servicer.CancelRun, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunResponse.SerializeToString, + ), + 'ListRunResults': grpc.unary_unary_rpc_method_handler( + servicer.ListRunResults, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsResponse.SerializeToString, + ), + 'WatchRun': grpc.unary_stream_rpc_method_handler( + servicer.WatchRun, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agentevals.api.v1alpha1.RunService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agentevals.api.v1alpha1.RunService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class RunService(object): + """Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every + method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. + """ + + @staticmethod + def SubmitRun(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.RunService/SubmitRun', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.SubmitRunResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetRun(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.RunService/GetRun', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.GetRunResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListRuns(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.RunService/ListRuns', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def CancelRun(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.RunService/CancelRun', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.CancelRunResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListRunResults(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.RunService/ListRunResults', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.ListRunResultsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def WatchRun(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/agentevals.api.v1alpha1.RunService/WatchRun', + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_runs__pb2.WatchRunResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/agentevals/api/v1alpha1/sessions_pb2.py b/src/agentevals/api/v1alpha1/sessions_pb2.py new file mode 100644 index 00000000..cc86fef2 --- /dev/null +++ b/src/agentevals/api/v1alpha1/sessions_pb2.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/sessions.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/sessions.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from agentevals.api.v1alpha1 import common_pb2 as agentevals_dot_api_dot_v1alpha1_dot_common__pb2 +from agentevals.api.v1alpha1 import views_pb2 as agentevals_dot_api_dot_v1alpha1_dot_views__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&agentevals/api/v1alpha1/sessions.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a#agentevals/api/v1alpha1/views.proto\"O\n\x13ListSessionsRequest\x12\x38\n\x04page\x18\x01 \x01(\x0b\x32$.agentevals.api.v1alpha1.PageRequestR\x04page\"\x96\x01\n\x14ListSessionsResponse\x12\x43\n\x08sessions\x18\x01 \x03(\x0b\x32\'.agentevals.api.v1alpha1.SessionSummaryR\x08sessions\x12\x39\n\x04page\x18\x02 \x01(\x0b\x32%.agentevals.api.v1alpha1.PageResponseR\x04page\"#\n\x11GetSessionRequest\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\"P\n\x12GetSessionResponse\x12:\n\x07session\x18\x01 \x01(\x0b\x32 .agentevals.api.v1alpha1.SessionR\x07session\";\n\x14WatchSessionsRequest\x12#\n\rinclude_turns\x18\x01 \x01(\x08R\x0cincludeTurns\"A\n\x0eSessionRemoved\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x1f\n\x0b\x61\x62sorbed_by\x18\x02 \x01(\tR\nabsorbedBy\"\x08\n\x06Synced\"\xdc\x01\n\x15WatchSessionsResponse\x12:\n\x06upsert\x18\x01 \x01(\x0b\x32 .agentevals.api.v1alpha1.SessionH\x00R\x06upsert\x12\x43\n\x07removed\x18\x02 \x01(\x0b\x32\'.agentevals.api.v1alpha1.SessionRemovedH\x00R\x07removed\x12\x39\n\x06synced\x18\x03 \x01(\x0b\x32\x1f.agentevals.api.v1alpha1.SyncedH\x00R\x06syncedB\x07\n\x05\x65vent2\xd6\x02\n\x0eSessionService\x12k\n\x0cListSessions\x12,.agentevals.api.v1alpha1.ListSessionsRequest\x1a-.agentevals.api.v1alpha1.ListSessionsResponse\x12\x65\n\nGetSession\x12*.agentevals.api.v1alpha1.GetSessionRequest\x1a+.agentevals.api.v1alpha1.GetSessionResponse\x12p\n\rWatchSessions\x12-.agentevals.api.v1alpha1.WatchSessionsRequest\x1a..agentevals.api.v1alpha1.WatchSessionsResponse0\x01\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.sessions_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_LISTSESSIONSREQUEST']._serialized_start=142 + _globals['_LISTSESSIONSREQUEST']._serialized_end=221 + _globals['_LISTSESSIONSRESPONSE']._serialized_start=224 + _globals['_LISTSESSIONSRESPONSE']._serialized_end=374 + _globals['_GETSESSIONREQUEST']._serialized_start=376 + _globals['_GETSESSIONREQUEST']._serialized_end=411 + _globals['_GETSESSIONRESPONSE']._serialized_start=413 + _globals['_GETSESSIONRESPONSE']._serialized_end=493 + _globals['_WATCHSESSIONSREQUEST']._serialized_start=495 + _globals['_WATCHSESSIONSREQUEST']._serialized_end=554 + _globals['_SESSIONREMOVED']._serialized_start=556 + _globals['_SESSIONREMOVED']._serialized_end=621 + _globals['_SYNCED']._serialized_start=623 + _globals['_SYNCED']._serialized_end=631 + _globals['_WATCHSESSIONSRESPONSE']._serialized_start=634 + _globals['_WATCHSESSIONSRESPONSE']._serialized_end=854 + _globals['_SESSIONSERVICE']._serialized_start=857 + _globals['_SESSIONSERVICE']._serialized_end=1199 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/sessions_pb2.pyi b/src/agentevals/api/v1alpha1/sessions_pb2.pyi new file mode 100644 index 00000000..d198ff78 --- /dev/null +++ b/src/agentevals/api/v1alpha1/sessions_pb2.pyi @@ -0,0 +1,63 @@ +from agentevals.api.v1alpha1 import common_pb2 as _common_pb2 +from agentevals.api.v1alpha1 import views_pb2 as _views_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class ListSessionsRequest(_message.Message): + __slots__ = ("page",) + PAGE_FIELD_NUMBER: _ClassVar[int] + page: _common_pb2.PageRequest + def __init__(self, page: _Optional[_Union[_common_pb2.PageRequest, _Mapping]] = ...) -> None: ... + +class ListSessionsResponse(_message.Message): + __slots__ = ("sessions", "page") + SESSIONS_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + sessions: _containers.RepeatedCompositeFieldContainer[_views_pb2.SessionSummary] + page: _common_pb2.PageResponse + def __init__(self, sessions: _Optional[_Iterable[_Union[_views_pb2.SessionSummary, _Mapping]]] = ..., page: _Optional[_Union[_common_pb2.PageResponse, _Mapping]] = ...) -> None: ... + +class GetSessionRequest(_message.Message): + __slots__ = ("id",) + ID_FIELD_NUMBER: _ClassVar[int] + id: str + def __init__(self, id: _Optional[str] = ...) -> None: ... + +class GetSessionResponse(_message.Message): + __slots__ = ("session",) + SESSION_FIELD_NUMBER: _ClassVar[int] + session: _views_pb2.Session + def __init__(self, session: _Optional[_Union[_views_pb2.Session, _Mapping]] = ...) -> None: ... + +class WatchSessionsRequest(_message.Message): + __slots__ = ("include_turns",) + INCLUDE_TURNS_FIELD_NUMBER: _ClassVar[int] + include_turns: bool + def __init__(self, include_turns: _Optional[bool] = ...) -> None: ... + +class SessionRemoved(_message.Message): + __slots__ = ("id", "absorbed_by") + ID_FIELD_NUMBER: _ClassVar[int] + ABSORBED_BY_FIELD_NUMBER: _ClassVar[int] + id: str + absorbed_by: str + def __init__(self, id: _Optional[str] = ..., absorbed_by: _Optional[str] = ...) -> None: ... + +class Synced(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class WatchSessionsResponse(_message.Message): + __slots__ = ("upsert", "removed", "synced") + UPSERT_FIELD_NUMBER: _ClassVar[int] + REMOVED_FIELD_NUMBER: _ClassVar[int] + SYNCED_FIELD_NUMBER: _ClassVar[int] + upsert: _views_pb2.Session + removed: SessionRemoved + synced: Synced + def __init__(self, upsert: _Optional[_Union[_views_pb2.Session, _Mapping]] = ..., removed: _Optional[_Union[SessionRemoved, _Mapping]] = ..., synced: _Optional[_Union[Synced, _Mapping]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/sessions_pb2_grpc.py b/src/agentevals/api/v1alpha1/sessions_pb2_grpc.py new file mode 100644 index 00000000..5ae4c19a --- /dev/null +++ b/src/agentevals/api/v1alpha1/sessions_pb2_grpc.py @@ -0,0 +1,171 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agentevals.api.v1alpha1 import sessions_pb2 as agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2 + + +class SessionServiceStub(object): + """Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in + memory and expire. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListSessions = channel.unary_unary( + '/agentevals.api.v1alpha1.SessionService/ListSessions', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsResponse.FromString, + _registered_method=True) + self.GetSession = channel.unary_unary( + '/agentevals.api.v1alpha1.SessionService/GetSession', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionResponse.FromString, + _registered_method=True) + self.WatchSessions = channel.unary_stream( + '/agentevals.api.v1alpha1.SessionService/WatchSessions', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsResponse.FromString, + _registered_method=True) + + +class SessionServiceServicer(object): + """Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in + memory and expire. + """ + + def ListSessions(self, request, context): + """Sessions newest first, summaries only. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetSession(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def WatchSessions(self, request, context): + """Every current session, then synced, then each change as it happens. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_SessionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListSessions': grpc.unary_unary_rpc_method_handler( + servicer.ListSessions, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsResponse.SerializeToString, + ), + 'GetSession': grpc.unary_unary_rpc_method_handler( + servicer.GetSession, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionResponse.SerializeToString, + ), + 'WatchSessions': grpc.unary_stream_rpc_method_handler( + servicer.WatchSessions, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agentevals.api.v1alpha1.SessionService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agentevals.api.v1alpha1.SessionService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class SessionService(object): + """Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in + memory and expire. + """ + + @staticmethod + def ListSessions(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.SessionService/ListSessions', + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.ListSessionsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetSession(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.SessionService/GetSession', + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.GetSessionResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def WatchSessions(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/agentevals.api.v1alpha1.SessionService/WatchSessions', + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_sessions__pb2.WatchSessionsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/agentevals/api/v1alpha1/system_pb2.py b/src/agentevals/api/v1alpha1/system_pb2.py new file mode 100644 index 00000000..95652701 --- /dev/null +++ b/src/agentevals/api/v1alpha1/system_pb2.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/system.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/system.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$agentevals/api/v1alpha1/system.proto\x12\x17\x61gentevals.api.v1alpha1\x1a\x1egoogle/protobuf/duration.proto\"\x13\n\x11GetVersionRequest\".\n\x12GetVersionResponse\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\"\x12\n\x10GetConfigRequest\"p\n\x05Ports\x12\x12\n\x04http\x18\x01 \x01(\x05R\x04http\x12\x1b\n\totlp_http\x18\x02 \x01(\x05R\x08otlpHttp\x12\x1b\n\totlp_grpc\x18\x03 \x01(\x05R\x08otlpGrpc\x12\x19\n\x08\x61pi_grpc\x18\x04 \x01(\x05R\x07\x61piGrpc\"\x8e\x03\n\nLiveLimits\x12!\n\x0cmax_sessions\x18\x01 \x01(\x05R\x0bmaxSessions\x12&\n\x0fspans_per_trace\x18\x02 \x01(\x05R\rspansPerTrace\x12*\n\x11spans_per_session\x18\x03 \x01(\x05R\x0fspansPerSession\x12,\n\x12traces_per_session\x18\x04 \x01(\x05R\x10tracesPerSession\x12\x1b\n\tmax_bytes\x18\x05 \x01(\x03R\x08maxBytes\x12\x44\n\x10\x63ompletion_grace\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0f\x63ompletionGrace\x12<\n\x0cidle_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0bidleTimeout\x12:\n\x0bsession_ttl\x18\x08 \x01(\x0b\x32\x19.google.protobuf.DurationR\nsessionTtl\"Y\n\tJudgeKeys\x12\x16\n\x06google\x18\x01 \x01(\x08R\x06google\x12\x1c\n\tanthropic\x18\x02 \x01(\x08R\tanthropic\x12\x16\n\x06openai\x18\x03 \x01(\x08R\x06openai\"\xb1\x02\n\x11GetConfigResponse\x12\x34\n\x05ports\x18\x01 \x01(\x0b\x32\x1e.agentevals.api.v1alpha1.PortsR\x05ports\x12\x44\n\x0blive_limits\x18\x02 \x01(\x0b\x32#.agentevals.api.v1alpha1.LiveLimitsR\nliveLimits\x12!\n\x0cruns_enabled\x18\x03 \x01(\x08R\x0brunsEnabled\x12:\n\x19\x65valuation_events_enabled\x18\x04 \x01(\x08R\x17\x65valuationEventsEnabled\x12\x41\n\njudge_keys\x18\x05 \x01(\x0b\x32\".agentevals.api.v1alpha1.JudgeKeysR\tjudgeKeys2\xda\x01\n\rSystemService\x12\x65\n\nGetVersion\x12*.agentevals.api.v1alpha1.GetVersionRequest\x1a+.agentevals.api.v1alpha1.GetVersionResponse\x12\x62\n\tGetConfig\x12).agentevals.api.v1alpha1.GetConfigRequest\x1a*.agentevals.api.v1alpha1.GetConfigResponseBQZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.system_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_GETVERSIONREQUEST']._serialized_start=97 + _globals['_GETVERSIONREQUEST']._serialized_end=116 + _globals['_GETVERSIONRESPONSE']._serialized_start=118 + _globals['_GETVERSIONRESPONSE']._serialized_end=164 + _globals['_GETCONFIGREQUEST']._serialized_start=166 + _globals['_GETCONFIGREQUEST']._serialized_end=184 + _globals['_PORTS']._serialized_start=186 + _globals['_PORTS']._serialized_end=298 + _globals['_LIVELIMITS']._serialized_start=301 + _globals['_LIVELIMITS']._serialized_end=699 + _globals['_JUDGEKEYS']._serialized_start=701 + _globals['_JUDGEKEYS']._serialized_end=790 + _globals['_GETCONFIGRESPONSE']._serialized_start=793 + _globals['_GETCONFIGRESPONSE']._serialized_end=1098 + _globals['_SYSTEMSERVICE']._serialized_start=1101 + _globals['_SYSTEMSERVICE']._serialized_end=1319 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/system_pb2.pyi b/src/agentevals/api/v1alpha1/system_pb2.pyi new file mode 100644 index 00000000..0bec1368 --- /dev/null +++ b/src/agentevals/api/v1alpha1/system_pb2.pyi @@ -0,0 +1,79 @@ +import datetime + +from google.protobuf import duration_pb2 as _duration_pb2 +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class GetVersionRequest(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class GetVersionResponse(_message.Message): + __slots__ = ("version",) + VERSION_FIELD_NUMBER: _ClassVar[int] + version: str + def __init__(self, version: _Optional[str] = ...) -> None: ... + +class GetConfigRequest(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class Ports(_message.Message): + __slots__ = ("http", "otlp_http", "otlp_grpc", "api_grpc") + HTTP_FIELD_NUMBER: _ClassVar[int] + OTLP_HTTP_FIELD_NUMBER: _ClassVar[int] + OTLP_GRPC_FIELD_NUMBER: _ClassVar[int] + API_GRPC_FIELD_NUMBER: _ClassVar[int] + http: int + otlp_http: int + otlp_grpc: int + api_grpc: int + def __init__(self, http: _Optional[int] = ..., otlp_http: _Optional[int] = ..., otlp_grpc: _Optional[int] = ..., api_grpc: _Optional[int] = ...) -> None: ... + +class LiveLimits(_message.Message): + __slots__ = ("max_sessions", "spans_per_trace", "spans_per_session", "traces_per_session", "max_bytes", "completion_grace", "idle_timeout", "session_ttl") + MAX_SESSIONS_FIELD_NUMBER: _ClassVar[int] + SPANS_PER_TRACE_FIELD_NUMBER: _ClassVar[int] + SPANS_PER_SESSION_FIELD_NUMBER: _ClassVar[int] + TRACES_PER_SESSION_FIELD_NUMBER: _ClassVar[int] + MAX_BYTES_FIELD_NUMBER: _ClassVar[int] + COMPLETION_GRACE_FIELD_NUMBER: _ClassVar[int] + IDLE_TIMEOUT_FIELD_NUMBER: _ClassVar[int] + SESSION_TTL_FIELD_NUMBER: _ClassVar[int] + max_sessions: int + spans_per_trace: int + spans_per_session: int + traces_per_session: int + max_bytes: int + completion_grace: _duration_pb2.Duration + idle_timeout: _duration_pb2.Duration + session_ttl: _duration_pb2.Duration + def __init__(self, max_sessions: _Optional[int] = ..., spans_per_trace: _Optional[int] = ..., spans_per_session: _Optional[int] = ..., traces_per_session: _Optional[int] = ..., max_bytes: _Optional[int] = ..., completion_grace: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., idle_timeout: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., session_ttl: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ...) -> None: ... + +class JudgeKeys(_message.Message): + __slots__ = ("google", "anthropic", "openai") + GOOGLE_FIELD_NUMBER: _ClassVar[int] + ANTHROPIC_FIELD_NUMBER: _ClassVar[int] + OPENAI_FIELD_NUMBER: _ClassVar[int] + google: bool + anthropic: bool + openai: bool + def __init__(self, google: _Optional[bool] = ..., anthropic: _Optional[bool] = ..., openai: _Optional[bool] = ...) -> None: ... + +class GetConfigResponse(_message.Message): + __slots__ = ("ports", "live_limits", "runs_enabled", "evaluation_events_enabled", "judge_keys") + PORTS_FIELD_NUMBER: _ClassVar[int] + LIVE_LIMITS_FIELD_NUMBER: _ClassVar[int] + RUNS_ENABLED_FIELD_NUMBER: _ClassVar[int] + EVALUATION_EVENTS_ENABLED_FIELD_NUMBER: _ClassVar[int] + JUDGE_KEYS_FIELD_NUMBER: _ClassVar[int] + ports: Ports + live_limits: LiveLimits + runs_enabled: bool + evaluation_events_enabled: bool + judge_keys: JudgeKeys + def __init__(self, ports: _Optional[_Union[Ports, _Mapping]] = ..., live_limits: _Optional[_Union[LiveLimits, _Mapping]] = ..., runs_enabled: _Optional[bool] = ..., evaluation_events_enabled: _Optional[bool] = ..., judge_keys: _Optional[_Union[JudgeKeys, _Mapping]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/system_pb2_grpc.py b/src/agentevals/api/v1alpha1/system_pb2_grpc.py new file mode 100644 index 00000000..0d7652ad --- /dev/null +++ b/src/agentevals/api/v1alpha1/system_pb2_grpc.py @@ -0,0 +1,120 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from agentevals.api.v1alpha1 import system_pb2 as agentevals_dot_api_dot_v1alpha1_dot_system__pb2 + + +class SystemServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetVersion = channel.unary_unary( + '/agentevals.api.v1alpha1.SystemService/GetVersion', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionResponse.FromString, + _registered_method=True) + self.GetConfig = channel.unary_unary( + '/agentevals.api.v1alpha1.SystemService/GetConfig', + request_serializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigRequest.SerializeToString, + response_deserializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigResponse.FromString, + _registered_method=True) + + +class SystemServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def GetVersion(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetConfig(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_SystemServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetVersion': grpc.unary_unary_rpc_method_handler( + servicer.GetVersion, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionResponse.SerializeToString, + ), + 'GetConfig': grpc.unary_unary_rpc_method_handler( + servicer.GetConfig, + request_deserializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigRequest.FromString, + response_serializer=agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'agentevals.api.v1alpha1.SystemService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('agentevals.api.v1alpha1.SystemService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class SystemService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetVersion(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.SystemService/GetVersion', + agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetVersionResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetConfig(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/agentevals.api.v1alpha1.SystemService/GetConfig', + agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigRequest.SerializeToString, + agentevals_dot_api_dot_v1alpha1_dot_system__pb2.GetConfigResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/agentevals/api/v1alpha1/views_pb2.py b/src/agentevals/api/v1alpha1/views_pb2.py new file mode 100644 index 00000000..442d50a0 --- /dev/null +++ b/src/agentevals/api/v1alpha1/views_pb2.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: agentevals/api/v1alpha1/views.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'agentevals/api/v1alpha1/views.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from opentelemetry.proto.resource.v1 import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 +from opentelemetry.proto.trace.v1 import trace_pb2 as opentelemetry_dot_proto_dot_trace_dot_v1_dot_trace__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#agentevals/api/v1alpha1/views.proto\x12\x17\x61gentevals.api.v1alpha1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\"\x88\x03\n\x05Usage\x12&\n\x0cinput_tokens\x18\x01 \x01(\x03H\x00R\x0binputTokens\x88\x01\x01\x12(\n\routput_tokens\x18\x02 \x01(\x03H\x01R\x0coutputTokens\x88\x01\x01\x12:\n\x17\x63\x61\x63he_read_input_tokens\x18\x03 \x01(\x03H\x02R\x14\x63\x61\x63heReadInputTokens\x88\x01\x01\x12<\n\x18\x63\x61\x63he_write_input_tokens\x18\x04 \x01(\x03H\x03R\x15\x63\x61\x63heWriteInputTokens\x88\x01\x01\x12;\n\x17reasoning_output_tokens\x18\x05 \x01(\x03H\x04R\x15reasoningOutputTokens\x88\x01\x01\x42\x0f\n\r_input_tokensB\x10\n\x0e_output_tokensB\x1a\n\x18_cache_read_input_tokensB\x1b\n\x19_cache_write_input_tokensB\x1a\n\x18_reasoning_output_tokens\"1\n\x03Gap\x12\x12\n\x04\x63ode\x18\x01 \x01(\tR\x04\x63ode\x12\x16\n\x06reason\x18\x02 \x01(\tR\x06reason\"\xf8\x02\n\x10\x45valuationResult\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12$\n\x0bscore_value\x18\x02 \x01(\x01H\x00R\nscoreValue\x88\x01\x01\x12$\n\x0bscore_label\x18\x03 \x01(\tH\x01R\nscoreLabel\x88\x01\x01\x12%\n\x0b\x65xplanation\x18\x04 \x01(\tH\x02R\x0b\x65xplanation\x88\x01\x01\x12\"\n\nerror_type\x18\x05 \x01(\tH\x03R\terrorType\x88\x01\x01\x12\x19\n\x08trace_id\x18\x06 \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x07 \x01(\tR\x06spanId\x12\x1f\n\x0bresponse_id\x18\x08 \x01(\tR\nresponseId\x12%\n\x0esource_service\x18\t \x01(\tR\rsourceServiceB\x0e\n\x0c_score_valueB\x0e\n\x0c_score_labelB\x0e\n\x0c_explanationB\r\n\x0b_error_type\"\xf7\x02\n\x0cToolCallView\x12\x17\n\x07span_id\x18\x01 \x01(\tR\x06spanId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1c\n\x07\x63\x61ll_id\x18\x03 \x01(\tH\x00R\x06\x63\x61llId\x88\x01\x01\x12\x34\n\targuments\x18\x04 \x01(\x0b\x32\x16.google.protobuf.ValueR\targuments\x12.\n\x06result\x18\x05 \x01(\x0b\x32\x16.google.protobuf.ValueR\x06result\x12\x19\n\x08is_error\x18\x06 \x01(\x08R\x07isError\x12\x1d\n\nerror_type\x18\x07 \x01(\tR\terrorType\x12\x39\n\nstarted_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x35\n\x08\x64uration\x18\t \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64urationB\n\n\x08_call_id\"\xa5\x04\n\rModelCallView\x12\x17\n\x07span_id\x18\x01 \x01(\tR\x06spanId\x12\x1c\n\toperation\x18\x02 \x01(\tR\toperation\x12\x1a\n\x08provider\x18\x03 \x01(\tR\x08provider\x12#\n\rrequest_model\x18\x04 \x01(\tR\x0crequestModel\x12%\n\x0eresponse_model\x18\x05 \x01(\tR\rresponseModel\x12\x1f\n\x0bresponse_id\x18\x06 \x01(\tR\nresponseId\x12\x34\n\x05usage\x18\x07 \x01(\x0b\x32\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x12%\n\x0e\x66inish_reasons\x18\x08 \x03(\tR\rfinishReasons\x12G\n\x06status\x18\t \x01(\x0e\x32/.opentelemetry.proto.trace.v1.Status.StatusCodeR\x06status\x12\x1d\n\nerror_type\x18\n \x01(\tR\terrorType\x12\x39\n\nstarted_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x35\n\x08\x64uration\x18\x0c \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64uration\x12\x1d\n\nagent_name\x18\r \x01(\tR\tagentName\"\xb3\x06\n\x08TurnView\x12\x14\n\x05index\x18\x01 \x01(\x05R\x05index\x12\x19\n\x08trace_id\x18\x02 \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x03 \x01(\tR\x06spanId\x12G\n\x06status\x18\x04 \x01(\x0e\x32/.opentelemetry.proto.trace.v1.Status.StatusCodeR\x06status\x12\x1d\n\nerror_type\x18\x05 \x01(\tR\terrorType\x12\x39\n\nstarted_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x35\n\x08\x64uration\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64uration\x12\x1d\n\nagent_name\x18\x08 \x01(\tR\tagentName\x12 \n\tuser_text\x18\t \x01(\tH\x00R\x08userText\x88\x01\x01\x12$\n\x0b\x61nswer_text\x18\n \x01(\tH\x01R\nanswerText\x88\x01\x01\x12)\n\x10\x63ontent_captured\x18\x0b \x01(\x08R\x0f\x63ontentCaptured\x12\x44\n\ntool_calls\x18\x0c \x03(\x0b\x32%.agentevals.api.v1alpha1.ToolCallViewR\ttoolCalls\x12G\n\x0bmodel_calls\x18\r \x03(\x0b\x32&.agentevals.api.v1alpha1.ModelCallViewR\nmodelCalls\x12\x34\n\x05usage\x18\x0e \x01(\x0b\x32\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x12\\\n\x14\x65xternal_evaluations\x18\x0f \x03(\x0b\x32).agentevals.api.v1alpha1.EvaluationResultR\x13\x65xternalEvaluations\x12\x30\n\x04gaps\x18\x10 \x03(\x0b\x32\x1c.agentevals.api.v1alpha1.GapR\x04gapsB\x0c\n\n_user_textB\x0e\n\x0c_answer_text\"\xae\x02\n\rSessionTotals\x12\x14\n\x05turns\x18\x01 \x01(\x05R\x05turns\x12!\n\x0c\x66\x61iled_turns\x18\x02 \x01(\x05R\x0b\x66\x61iledTurns\x12\x1f\n\x0bmodel_calls\x18\x03 \x01(\x05R\nmodelCalls\x12\x1d\n\ntool_calls\x18\x04 \x01(\x05R\ttoolCalls\x12\x34\n\x05usage\x18\x05 \x01(\x0b\x32\x1e.agentevals.api.v1alpha1.UsageR\x05usage\x12\x33\n\x16model_calls_with_usage\x18\x06 \x01(\x05R\x13modelCallsWithUsage\x12\x39\n\x19tool_calls_with_arguments\x18\x07 \x01(\x05R\x16toolCallsWithArguments\"\xc5\x06\n\x0eSessionSummary\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x19\n\x08short_id\x18\x02 \x01(\tR\x07shortId\x12\x14\n\x05\x61gent\x18\x03 \x01(\tR\x05\x61gent\x12#\n\ragent_version\x18\x04 \x01(\tR\x0c\x61gentVersion\x12;\n\x05state\x18\x05 \x01(\x0e\x32%.agentevals.api.v1alpha1.SessionStateR\x05state\x12\x39\n\nstarted_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x35\n\x08\x65nded_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndedAt\x12\x35\n\x08\x64uration\x18\x08 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64uration\x12+\n\x0f\x66irst_user_text\x18\t \x01(\tH\x00R\rfirstUserText\x88\x01\x01\x12P\n\x0f\x63ontent_capture\x18\n \x01(\x0e\x32\'.agentevals.api.v1alpha1.ContentCaptureR\x0e\x63ontentCapture\x12>\n\x06totals\x18\x0b \x01(\x0b\x32&.agentevals.api.v1alpha1.SessionTotalsR\x06totals\x12\x16\n\x06models\x18\x0c \x03(\tR\x06models\x12\x45\n\x08resource\x18\r \x01(\x0b\x32).opentelemetry.proto.resource.v1.ResourceR\x08resource\x12\'\n\x0f\x63onversation_id\x18\x0e \x01(\tR\x0e\x63onversationId\x12\x1b\n\ttrace_ids\x18\x0f \x03(\tR\x08traceIds\x12\x1e\n\x0b\x65val_set_id\x18\x10 \x01(\tR\tevalSetId\x12\x1d\n\nspan_count\x18\x11 \x01(\x05R\tspanCount\x12\x30\n\x04gaps\x18\x12 \x03(\x0b\x32\x1c.agentevals.api.v1alpha1.GapR\x04gapsB\x12\n\x10_first_user_text\"\x85\x01\n\x07Session\x12\x41\n\x07summary\x18\x01 \x01(\x0b\x32\'.agentevals.api.v1alpha1.SessionSummaryR\x07summary\x12\x37\n\x05turns\x18\x02 \x03(\x0b\x32!.agentevals.api.v1alpha1.TurnViewR\x05turns\"\xd5\x01\n\x10\x43onversationView\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\'\n\x0f\x63onversation_id\x18\x02 \x01(\tR\x0e\x63onversationId\x12\x1b\n\ttrace_ids\x18\x03 \x03(\tR\x08traceIds\x12\x37\n\x05turns\x18\x04 \x03(\x0b\x32!.agentevals.api.v1alpha1.TurnViewR\x05turns\x12\x30\n\x04gaps\x18\x05 \x03(\x0b\x32\x1c.agentevals.api.v1alpha1.GapR\x04gaps*a\n\x0cSessionState\x12\x1d\n\x19SESSION_STATE_UNSPECIFIED\x10\x00\x12\x16\n\x12SESSION_STATE_LIVE\x10\x01\x12\x1a\n\x16SESSION_STATE_COMPLETE\x10\x02*\x7f\n\x0e\x43ontentCapture\x12\x1f\n\x1b\x43ONTENT_CAPTURE_UNSPECIFIED\x10\x00\x12\x16\n\x12\x43ONTENT_CAPTURE_ON\x10\x01\x12\x17\n\x13\x43ONTENT_CAPTURE_OFF\x10\x02\x12\x1b\n\x17\x43ONTENT_CAPTURE_PARTIAL\x10\x03\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'agentevals.api.v1alpha1.views_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' + _globals['_SESSIONSTATE']._serialized_start=4323 + _globals['_SESSIONSTATE']._serialized_end=4420 + _globals['_CONTENTCAPTURE']._serialized_start=4422 + _globals['_CONTENTCAPTURE']._serialized_end=4549 + _globals['_USAGE']._serialized_start=250 + _globals['_USAGE']._serialized_end=642 + _globals['_GAP']._serialized_start=644 + _globals['_GAP']._serialized_end=693 + _globals['_EVALUATIONRESULT']._serialized_start=696 + _globals['_EVALUATIONRESULT']._serialized_end=1072 + _globals['_TOOLCALLVIEW']._serialized_start=1075 + _globals['_TOOLCALLVIEW']._serialized_end=1450 + _globals['_MODELCALLVIEW']._serialized_start=1453 + _globals['_MODELCALLVIEW']._serialized_end=2002 + _globals['_TURNVIEW']._serialized_start=2005 + _globals['_TURNVIEW']._serialized_end=2824 + _globals['_SESSIONTOTALS']._serialized_start=2827 + _globals['_SESSIONTOTALS']._serialized_end=3129 + _globals['_SESSIONSUMMARY']._serialized_start=3132 + _globals['_SESSIONSUMMARY']._serialized_end=3969 + _globals['_SESSION']._serialized_start=3972 + _globals['_SESSION']._serialized_end=4105 + _globals['_CONVERSATIONVIEW']._serialized_start=4108 + _globals['_CONVERSATIONVIEW']._serialized_end=4321 +# @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/views_pb2.pyi b/src/agentevals/api/v1alpha1/views_pb2.pyi new file mode 100644 index 00000000..75e61aa4 --- /dev/null +++ b/src/agentevals/api/v1alpha1/views_pb2.pyi @@ -0,0 +1,247 @@ +import datetime + +from google.protobuf import duration_pb2 as _duration_pb2 +from google.protobuf import struct_pb2 as _struct_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from opentelemetry.proto.resource.v1 import resource_pb2 as _resource_pb2 +from opentelemetry.proto.trace.v1 import trace_pb2 as _trace_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class SessionState(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + SESSION_STATE_UNSPECIFIED: _ClassVar[SessionState] + SESSION_STATE_LIVE: _ClassVar[SessionState] + SESSION_STATE_COMPLETE: _ClassVar[SessionState] + +class ContentCapture(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + CONTENT_CAPTURE_UNSPECIFIED: _ClassVar[ContentCapture] + CONTENT_CAPTURE_ON: _ClassVar[ContentCapture] + CONTENT_CAPTURE_OFF: _ClassVar[ContentCapture] + CONTENT_CAPTURE_PARTIAL: _ClassVar[ContentCapture] +SESSION_STATE_UNSPECIFIED: SessionState +SESSION_STATE_LIVE: SessionState +SESSION_STATE_COMPLETE: SessionState +CONTENT_CAPTURE_UNSPECIFIED: ContentCapture +CONTENT_CAPTURE_ON: ContentCapture +CONTENT_CAPTURE_OFF: ContentCapture +CONTENT_CAPTURE_PARTIAL: ContentCapture + +class Usage(_message.Message): + __slots__ = ("input_tokens", "output_tokens", "cache_read_input_tokens", "cache_write_input_tokens", "reasoning_output_tokens") + INPUT_TOKENS_FIELD_NUMBER: _ClassVar[int] + OUTPUT_TOKENS_FIELD_NUMBER: _ClassVar[int] + CACHE_READ_INPUT_TOKENS_FIELD_NUMBER: _ClassVar[int] + CACHE_WRITE_INPUT_TOKENS_FIELD_NUMBER: _ClassVar[int] + REASONING_OUTPUT_TOKENS_FIELD_NUMBER: _ClassVar[int] + input_tokens: int + output_tokens: int + cache_read_input_tokens: int + cache_write_input_tokens: int + reasoning_output_tokens: int + def __init__(self, input_tokens: _Optional[int] = ..., output_tokens: _Optional[int] = ..., cache_read_input_tokens: _Optional[int] = ..., cache_write_input_tokens: _Optional[int] = ..., reasoning_output_tokens: _Optional[int] = ...) -> None: ... + +class Gap(_message.Message): + __slots__ = ("code", "reason") + CODE_FIELD_NUMBER: _ClassVar[int] + REASON_FIELD_NUMBER: _ClassVar[int] + code: str + reason: str + def __init__(self, code: _Optional[str] = ..., reason: _Optional[str] = ...) -> None: ... + +class EvaluationResult(_message.Message): + __slots__ = ("name", "score_value", "score_label", "explanation", "error_type", "trace_id", "span_id", "response_id", "source_service") + NAME_FIELD_NUMBER: _ClassVar[int] + SCORE_VALUE_FIELD_NUMBER: _ClassVar[int] + SCORE_LABEL_FIELD_NUMBER: _ClassVar[int] + EXPLANATION_FIELD_NUMBER: _ClassVar[int] + ERROR_TYPE_FIELD_NUMBER: _ClassVar[int] + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + RESPONSE_ID_FIELD_NUMBER: _ClassVar[int] + SOURCE_SERVICE_FIELD_NUMBER: _ClassVar[int] + name: str + score_value: float + score_label: str + explanation: str + error_type: str + trace_id: str + span_id: str + response_id: str + source_service: str + def __init__(self, name: _Optional[str] = ..., score_value: _Optional[float] = ..., score_label: _Optional[str] = ..., explanation: _Optional[str] = ..., error_type: _Optional[str] = ..., trace_id: _Optional[str] = ..., span_id: _Optional[str] = ..., response_id: _Optional[str] = ..., source_service: _Optional[str] = ...) -> None: ... + +class ToolCallView(_message.Message): + __slots__ = ("span_id", "name", "call_id", "arguments", "result", "is_error", "error_type", "started_at", "duration") + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + CALL_ID_FIELD_NUMBER: _ClassVar[int] + ARGUMENTS_FIELD_NUMBER: _ClassVar[int] + RESULT_FIELD_NUMBER: _ClassVar[int] + IS_ERROR_FIELD_NUMBER: _ClassVar[int] + ERROR_TYPE_FIELD_NUMBER: _ClassVar[int] + STARTED_AT_FIELD_NUMBER: _ClassVar[int] + DURATION_FIELD_NUMBER: _ClassVar[int] + span_id: str + name: str + call_id: str + arguments: _struct_pb2.Value + result: _struct_pb2.Value + is_error: bool + error_type: str + started_at: _timestamp_pb2.Timestamp + duration: _duration_pb2.Duration + def __init__(self, span_id: _Optional[str] = ..., name: _Optional[str] = ..., call_id: _Optional[str] = ..., arguments: _Optional[_Union[_struct_pb2.Value, _Mapping]] = ..., result: _Optional[_Union[_struct_pb2.Value, _Mapping]] = ..., is_error: _Optional[bool] = ..., error_type: _Optional[str] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., duration: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ...) -> None: ... + +class ModelCallView(_message.Message): + __slots__ = ("span_id", "operation", "provider", "request_model", "response_model", "response_id", "usage", "finish_reasons", "status", "error_type", "started_at", "duration", "agent_name") + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + OPERATION_FIELD_NUMBER: _ClassVar[int] + PROVIDER_FIELD_NUMBER: _ClassVar[int] + REQUEST_MODEL_FIELD_NUMBER: _ClassVar[int] + RESPONSE_MODEL_FIELD_NUMBER: _ClassVar[int] + RESPONSE_ID_FIELD_NUMBER: _ClassVar[int] + USAGE_FIELD_NUMBER: _ClassVar[int] + FINISH_REASONS_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + ERROR_TYPE_FIELD_NUMBER: _ClassVar[int] + STARTED_AT_FIELD_NUMBER: _ClassVar[int] + DURATION_FIELD_NUMBER: _ClassVar[int] + AGENT_NAME_FIELD_NUMBER: _ClassVar[int] + span_id: str + operation: str + provider: str + request_model: str + response_model: str + response_id: str + usage: Usage + finish_reasons: _containers.RepeatedScalarFieldContainer[str] + status: _trace_pb2.Status.StatusCode + error_type: str + started_at: _timestamp_pb2.Timestamp + duration: _duration_pb2.Duration + agent_name: str + def __init__(self, span_id: _Optional[str] = ..., operation: _Optional[str] = ..., provider: _Optional[str] = ..., request_model: _Optional[str] = ..., response_model: _Optional[str] = ..., response_id: _Optional[str] = ..., usage: _Optional[_Union[Usage, _Mapping]] = ..., finish_reasons: _Optional[_Iterable[str]] = ..., status: _Optional[_Union[_trace_pb2.Status.StatusCode, str]] = ..., error_type: _Optional[str] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., duration: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., agent_name: _Optional[str] = ...) -> None: ... + +class TurnView(_message.Message): + __slots__ = ("index", "trace_id", "span_id", "status", "error_type", "started_at", "duration", "agent_name", "user_text", "answer_text", "content_captured", "tool_calls", "model_calls", "usage", "external_evaluations", "gaps") + INDEX_FIELD_NUMBER: _ClassVar[int] + TRACE_ID_FIELD_NUMBER: _ClassVar[int] + SPAN_ID_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + ERROR_TYPE_FIELD_NUMBER: _ClassVar[int] + STARTED_AT_FIELD_NUMBER: _ClassVar[int] + DURATION_FIELD_NUMBER: _ClassVar[int] + AGENT_NAME_FIELD_NUMBER: _ClassVar[int] + USER_TEXT_FIELD_NUMBER: _ClassVar[int] + ANSWER_TEXT_FIELD_NUMBER: _ClassVar[int] + CONTENT_CAPTURED_FIELD_NUMBER: _ClassVar[int] + TOOL_CALLS_FIELD_NUMBER: _ClassVar[int] + MODEL_CALLS_FIELD_NUMBER: _ClassVar[int] + USAGE_FIELD_NUMBER: _ClassVar[int] + EXTERNAL_EVALUATIONS_FIELD_NUMBER: _ClassVar[int] + GAPS_FIELD_NUMBER: _ClassVar[int] + index: int + trace_id: str + span_id: str + status: _trace_pb2.Status.StatusCode + error_type: str + started_at: _timestamp_pb2.Timestamp + duration: _duration_pb2.Duration + agent_name: str + user_text: str + answer_text: str + content_captured: bool + tool_calls: _containers.RepeatedCompositeFieldContainer[ToolCallView] + model_calls: _containers.RepeatedCompositeFieldContainer[ModelCallView] + usage: Usage + external_evaluations: _containers.RepeatedCompositeFieldContainer[EvaluationResult] + gaps: _containers.RepeatedCompositeFieldContainer[Gap] + def __init__(self, index: _Optional[int] = ..., trace_id: _Optional[str] = ..., span_id: _Optional[str] = ..., status: _Optional[_Union[_trace_pb2.Status.StatusCode, str]] = ..., error_type: _Optional[str] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., duration: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., agent_name: _Optional[str] = ..., user_text: _Optional[str] = ..., answer_text: _Optional[str] = ..., content_captured: _Optional[bool] = ..., tool_calls: _Optional[_Iterable[_Union[ToolCallView, _Mapping]]] = ..., model_calls: _Optional[_Iterable[_Union[ModelCallView, _Mapping]]] = ..., usage: _Optional[_Union[Usage, _Mapping]] = ..., external_evaluations: _Optional[_Iterable[_Union[EvaluationResult, _Mapping]]] = ..., gaps: _Optional[_Iterable[_Union[Gap, _Mapping]]] = ...) -> None: ... + +class SessionTotals(_message.Message): + __slots__ = ("turns", "failed_turns", "model_calls", "tool_calls", "usage", "model_calls_with_usage", "tool_calls_with_arguments") + TURNS_FIELD_NUMBER: _ClassVar[int] + FAILED_TURNS_FIELD_NUMBER: _ClassVar[int] + MODEL_CALLS_FIELD_NUMBER: _ClassVar[int] + TOOL_CALLS_FIELD_NUMBER: _ClassVar[int] + USAGE_FIELD_NUMBER: _ClassVar[int] + MODEL_CALLS_WITH_USAGE_FIELD_NUMBER: _ClassVar[int] + TOOL_CALLS_WITH_ARGUMENTS_FIELD_NUMBER: _ClassVar[int] + turns: int + failed_turns: int + model_calls: int + tool_calls: int + usage: Usage + model_calls_with_usage: int + tool_calls_with_arguments: int + def __init__(self, turns: _Optional[int] = ..., failed_turns: _Optional[int] = ..., model_calls: _Optional[int] = ..., tool_calls: _Optional[int] = ..., usage: _Optional[_Union[Usage, _Mapping]] = ..., model_calls_with_usage: _Optional[int] = ..., tool_calls_with_arguments: _Optional[int] = ...) -> None: ... + +class SessionSummary(_message.Message): + __slots__ = ("id", "short_id", "agent", "agent_version", "state", "started_at", "ended_at", "duration", "first_user_text", "content_capture", "totals", "models", "resource", "conversation_id", "trace_ids", "eval_set_id", "span_count", "gaps") + ID_FIELD_NUMBER: _ClassVar[int] + SHORT_ID_FIELD_NUMBER: _ClassVar[int] + AGENT_FIELD_NUMBER: _ClassVar[int] + AGENT_VERSION_FIELD_NUMBER: _ClassVar[int] + STATE_FIELD_NUMBER: _ClassVar[int] + STARTED_AT_FIELD_NUMBER: _ClassVar[int] + ENDED_AT_FIELD_NUMBER: _ClassVar[int] + DURATION_FIELD_NUMBER: _ClassVar[int] + FIRST_USER_TEXT_FIELD_NUMBER: _ClassVar[int] + CONTENT_CAPTURE_FIELD_NUMBER: _ClassVar[int] + TOTALS_FIELD_NUMBER: _ClassVar[int] + MODELS_FIELD_NUMBER: _ClassVar[int] + RESOURCE_FIELD_NUMBER: _ClassVar[int] + CONVERSATION_ID_FIELD_NUMBER: _ClassVar[int] + TRACE_IDS_FIELD_NUMBER: _ClassVar[int] + EVAL_SET_ID_FIELD_NUMBER: _ClassVar[int] + SPAN_COUNT_FIELD_NUMBER: _ClassVar[int] + GAPS_FIELD_NUMBER: _ClassVar[int] + id: str + short_id: str + agent: str + agent_version: str + state: SessionState + started_at: _timestamp_pb2.Timestamp + ended_at: _timestamp_pb2.Timestamp + duration: _duration_pb2.Duration + first_user_text: str + content_capture: ContentCapture + totals: SessionTotals + models: _containers.RepeatedScalarFieldContainer[str] + resource: _resource_pb2.Resource + conversation_id: str + trace_ids: _containers.RepeatedScalarFieldContainer[str] + eval_set_id: str + span_count: int + gaps: _containers.RepeatedCompositeFieldContainer[Gap] + def __init__(self, id: _Optional[str] = ..., short_id: _Optional[str] = ..., agent: _Optional[str] = ..., agent_version: _Optional[str] = ..., state: _Optional[_Union[SessionState, str]] = ..., started_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., ended_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., duration: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ..., first_user_text: _Optional[str] = ..., content_capture: _Optional[_Union[ContentCapture, str]] = ..., totals: _Optional[_Union[SessionTotals, _Mapping]] = ..., models: _Optional[_Iterable[str]] = ..., resource: _Optional[_Union[_resource_pb2.Resource, _Mapping]] = ..., conversation_id: _Optional[str] = ..., trace_ids: _Optional[_Iterable[str]] = ..., eval_set_id: _Optional[str] = ..., span_count: _Optional[int] = ..., gaps: _Optional[_Iterable[_Union[Gap, _Mapping]]] = ...) -> None: ... + +class Session(_message.Message): + __slots__ = ("summary", "turns") + SUMMARY_FIELD_NUMBER: _ClassVar[int] + TURNS_FIELD_NUMBER: _ClassVar[int] + summary: SessionSummary + turns: _containers.RepeatedCompositeFieldContainer[TurnView] + def __init__(self, summary: _Optional[_Union[SessionSummary, _Mapping]] = ..., turns: _Optional[_Iterable[_Union[TurnView, _Mapping]]] = ...) -> None: ... + +class ConversationView(_message.Message): + __slots__ = ("key", "conversation_id", "trace_ids", "turns", "gaps") + KEY_FIELD_NUMBER: _ClassVar[int] + CONVERSATION_ID_FIELD_NUMBER: _ClassVar[int] + TRACE_IDS_FIELD_NUMBER: _ClassVar[int] + TURNS_FIELD_NUMBER: _ClassVar[int] + GAPS_FIELD_NUMBER: _ClassVar[int] + key: str + conversation_id: str + trace_ids: _containers.RepeatedScalarFieldContainer[str] + turns: _containers.RepeatedCompositeFieldContainer[TurnView] + gaps: _containers.RepeatedCompositeFieldContainer[Gap] + def __init__(self, key: _Optional[str] = ..., conversation_id: _Optional[str] = ..., trace_ids: _Optional[_Iterable[str]] = ..., turns: _Optional[_Iterable[_Union[TurnView, _Mapping]]] = ..., gaps: _Optional[_Iterable[_Union[Gap, _Mapping]]] = ...) -> None: ... diff --git a/src/agentevals/api/v1alpha1/views_pb2_grpc.py b/src/agentevals/api/v1alpha1/views_pb2_grpc.py new file mode 100644 index 00000000..2daafffe --- /dev/null +++ b/src/agentevals/api/v1alpha1/views_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/src/agentevals/api/views.py b/src/agentevals/api/views.py new file mode 100644 index 00000000..a74f4fa2 --- /dev/null +++ b/src/agentevals/api/views.py @@ -0,0 +1,341 @@ +"""Views of sessions and conversations as ``agentevals.api.v1alpha1`` messages. + +Built from the GenAI model, so every client renders the same turns without deriving them. The +builders are pure and need no server: callers pass copied facts and frozen spans, never live +store objects, so a builder can run in a worker thread while ingest goes on. + +Every view encodes as protojson: texts and tool payloads are capped, non finite numbers become +null and bytes become base64 strings, each with a gap where something was changed. +""" + +from __future__ import annotations + +import base64 +import json +import math +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from google.protobuf import json_format, struct_pb2 +from opentelemetry.proto.resource.v1 import resource_pb2 +from opentelemetry.proto.trace.v1 import trace_pb2 + +from ..genai.messages import text_of +from ..genai.model import Conversation, ExternalEvaluation, Gap, LlmCall, ToolCall, Turn, Usage +from ..otel.encode import attributes_to_json +from ..otel.model import Resource, Span, attr_str +from .v1alpha1 import views_pb2 + +MAX_TEXT_CHARS = 64 * 1024 +MAX_VALUE_BYTES = 64 * 1024 +MAX_VALUE_DEPTH = 32 +# A JSON number is a double, exact for integers up to 2^53. +MAX_SAFE_INTEGER = 2**53 +SHORT_ID_LENGTH = 6 +# Generated ids (UUIDs, trace ids) are often time ordered, so only their tails tell them apart. +_GENERATED_ID = re.compile( + r"[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|[0-9a-fA-F]{16,}" +) + +GAP_TRUNCATED = "truncated" +GAP_NO_TURNS = "no_turns" + +AGENT_NAME = "gen_ai.agent.name" +AGENT_VERSION = "gen_ai.agent.version" +SERVICE_NAME = "service.name" +SERVICE_VERSION = "service.version" + +_STATUS = { + "unset": trace_pb2.Status.STATUS_CODE_UNSET, + "ok": trace_pb2.Status.STATUS_CODE_OK, + "error": trace_pb2.Status.STATUS_CODE_ERROR, +} + +SpanIndex = Mapping[tuple[str, str], Span] + + +@dataclass(frozen=True, slots=True) +class SessionFacts: + """What a session summary needs from the store, copied while no ingest can change it.""" + + session_id: str + received_at: datetime + is_complete: bool + trace_ids: tuple[str, ...] + span_count: int + eval_set_id: str | None = None + + +def session_view( + facts: SessionFacts, + conversation: Conversation | None = None, + spans: SpanIndex | None = None, + *, + include_turns: bool = True, +) -> views_pb2.Session: + """The session's summary and turns. Without a conversation, a summary from the facts alone.""" + spans = spans or {} + turns = [turn_view(t, spans) for t in conversation.turns] if conversation else [] + session = views_pb2.Session(summary=_summary(facts, conversation, spans, turns)) + if include_turns: + session.turns.extend(turns) + return session + + +def conversation_view(conversation: Conversation, spans: SpanIndex) -> views_pb2.ConversationView: + return views_pb2.ConversationView( + key=conversation.key, + conversation_id=conversation.conversation_id or "", + trace_ids=conversation.trace_ids, + turns=[turn_view(t, spans) for t in conversation.turns], + ) + + +def turn_view(turn: Turn, spans: SpanIndex) -> views_pb2.TurnView: + gaps: list[Gap] = list(turn.warnings) + view = views_pb2.TurnView( + index=turn.index, + trace_id=turn.ref.trace_id, + span_id=turn.ref.span_id, + status=_STATUS.get(turn.status, trace_pb2.Status.STATUS_CODE_UNSET), + error_type=turn.error_type or "", + agent_name=turn.agent_name or "", + content_captured=turn.content_captured, + tool_calls=[_tool_call(t, spans, gaps) for t in turn.tool_calls], + model_calls=[_model_call(c) for c in turn.llm_calls], + usage=_usage(turn.usage), + external_evaluations=[_external_evaluation(e) for e in turn.external_evaluations], + ) + _set_times(view, turn.start_ns, turn.end_ns) + user_text = text_of(turn.user_input) + if user_text is not None: + view.user_text = _capped_text(user_text, "user text", gaps) + answer_text = text_of(turn.final_output) + if answer_text is not None: + view.answer_text = _capped_text(answer_text, "answer text", gaps) + view.gaps.extend(views_pb2.Gap(code=g.code, reason=g.reason) for g in dict.fromkeys(gaps)) + return view + + +def _summary( + facts: SessionFacts, + conversation: Conversation | None, + spans: SpanIndex, + turns: list[views_pb2.TurnView], +) -> views_pb2.SessionSummary: + source = conversation.turns if conversation else [] + calls = [c for t in source for c in t.llm_calls] + tools = [tc for t in source for tc in t.tool_calls] + anchor = _anchor_span(source, spans) + resource = anchor.resource if anchor is not None else None + resource_attrs = resource.attributes if resource is not None else {} + + summary = views_pb2.SessionSummary( + id=facts.session_id, + short_id=_short_id(facts.session_id), + agent=_str(resource_attrs.get(AGENT_NAME)) + or next((t.agent_name for t in source if t.agent_name), None) + or _str(resource_attrs.get(SERVICE_NAME)), + agent_version=_str(resource_attrs.get(AGENT_VERSION)) + or (attr_str(anchor.attributes, AGENT_VERSION) if anchor is not None else None) + or _str(resource_attrs.get(SERVICE_VERSION)), + state=views_pb2.SESSION_STATE_COMPLETE if facts.is_complete else views_pb2.SESSION_STATE_LIVE, + content_capture=_content_capture(source), + totals=views_pb2.SessionTotals( + turns=len(source), + failed_turns=sum(1 for t in source if t.status == "error"), + model_calls=len(calls), + tool_calls=len(tools), + usage=_usage(_sum_usage(t.usage for t in source)), + model_calls_with_usage=sum(1 for c in calls if c.usage is not None), + tool_calls_with_arguments=sum(1 for tc in tools if tc.arguments is not None), + ), + models=list(dict.fromkeys(m for c in calls if (m := c.response_model or c.request_model))), + conversation_id=(conversation.conversation_id if conversation else None) or "", + trace_ids=list(conversation.trace_ids if conversation and conversation.trace_ids else facts.trace_ids), + eval_set_id=facts.eval_set_id or "", + span_count=facts.span_count, + ) + if resource is not None: + summary.resource.CopyFrom(_resource(resource)) + if source: + _set_times(summary, min(t.start_ns for t in source), max(t.end_ns for t in source)) + else: + summary.started_at.FromDatetime(facts.received_at) + if turns and turns[0].HasField("user_text"): + summary.first_user_text = turns[0].user_text + if facts.is_complete and facts.span_count and not source: + summary.gaps.append(views_pb2.Gap(code=GAP_NO_TURNS, reason="no span forms a GenAI turn")) + return summary + + +def _short_id(session_id: str) -> str: + return session_id[-SHORT_ID_LENGTH:] if _GENERATED_ID.fullmatch(session_id) else session_id + + +def _anchor_span(turns: list[Turn], spans: SpanIndex) -> Span | None: + """The first turn's anchor span, else the earliest span: the service that produced the session.""" + for turn in turns: + span = spans.get((turn.ref.trace_id, turn.ref.span_id)) + if span is not None: + return span + return min(spans.values(), key=lambda s: s.start_time_unix_nano, default=None) + + +def _content_capture(turns: list[Turn]) -> int: + captured = {t.content_captured for t in turns} + if not captured: + return views_pb2.CONTENT_CAPTURE_UNSPECIFIED + if captured == {True}: + return views_pb2.CONTENT_CAPTURE_ON + if captured == {False}: + return views_pb2.CONTENT_CAPTURE_OFF + return views_pb2.CONTENT_CAPTURE_PARTIAL + + +def _tool_call(tool: ToolCall, spans: SpanIndex, gaps: list[Gap]) -> views_pb2.ToolCallView: + view = views_pb2.ToolCallView( + span_id=tool.ref.span_id if tool.ref else "", + name=tool.name, + is_error=tool.is_error, + error_type=tool.error_type or "", + ) + if tool.call_id is not None: + view.call_id = tool.call_id + if tool.arguments is not None: + view.arguments.CopyFrom(_value(tool.arguments, f"tool {tool.name} arguments", gaps)) + if tool.result is not None: + view.result.CopyFrom(_value(tool.result, f"tool {tool.name} result", gaps)) + span = spans.get((tool.ref.trace_id, tool.ref.span_id)) if tool.ref else None + if span is not None: + _set_times(view, span.start_time_unix_nano, span.end_time_unix_nano) + elif tool.start_ns: + view.started_at.FromNanoseconds(tool.start_ns) + return view + + +def _model_call(call: LlmCall) -> views_pb2.ModelCallView: + view = views_pb2.ModelCallView( + span_id=call.ref.span_id, + operation=call.operation, + provider=call.provider or "", + request_model=call.request_model or "", + response_model=call.response_model or "", + response_id=call.response_id or "", + usage=_usage(call.usage), + finish_reasons=call.finish_reasons, + status=_STATUS.get(call.status, trace_pb2.Status.STATUS_CODE_UNSET), + error_type=call.error_type or "", + agent_name=call.agent_name or "", + ) + _set_times(view, call.start_ns, call.end_ns) + return view + + +def _external_evaluation(evaluation: ExternalEvaluation) -> views_pb2.EvaluationResult: + view = views_pb2.EvaluationResult( + name=evaluation.name, + trace_id=evaluation.ref.trace_id, + span_id=evaluation.ref.span_id, + source_service=evaluation.source_service or "", + ) + if evaluation.score_value is not None and math.isfinite(evaluation.score_value): + view.score_value = evaluation.score_value + if evaluation.score_label is not None: + view.score_label = evaluation.score_label + if evaluation.explanation is not None: + view.explanation = evaluation.explanation + return view + + +def _usage(usage: Usage | None) -> views_pb2.Usage: + view = views_pb2.Usage() + if usage is not None: + for name, value in usage.model_dump().items(): + if value is not None: + setattr(view, name, value) + return view + + +def _sum_usage(usages: Iterable[Usage]) -> Usage: + total = Usage() + for usage in usages: + total = total + usage + return total + + +def _resource(resource: Resource) -> resource_pb2.Resource: + """The resource through the OTLP/JSON encoder, so both encodings agree on every value.""" + out = resource_pb2.Resource(dropped_attributes_count=resource.dropped_attributes_count) + json_format.ParseDict({"attributes": attributes_to_json(resource.attributes)}, out) + return out + + +def _set_times(view: Any, start_ns: int | None, end_ns: int | None) -> None: + if not start_ns: + return + view.started_at.FromNanoseconds(start_ns) + if end_ns and end_ns >= start_ns: + if "ended_at" in view.DESCRIPTOR.fields_by_name: + view.ended_at.FromNanoseconds(end_ns) + view.duration.FromNanoseconds(end_ns - start_ns) + + +def _capped_text(text: str, what: str, gaps: list[Gap]) -> str: + if len(text) <= MAX_TEXT_CHARS: + return text + gaps.append(Gap(code=GAP_TRUNCATED, reason=f"{what} cut to {MAX_TEXT_CHARS} characters")) + return text[:MAX_TEXT_CHARS] + + +def _value(obj: Any, what: str, gaps: list[Gap]) -> struct_pb2.Value: + changed: list[str] = [] + safe = _json_safe(obj, 0, changed) + encoded = json.dumps(safe, ensure_ascii=False, separators=(",", ":")) + if len(encoded.encode()) > MAX_VALUE_BYTES: + safe = encoded.encode()[:MAX_VALUE_BYTES].decode(errors="ignore") + changed.append(f"cut to {MAX_VALUE_BYTES} bytes of JSON text") + if changed: + gaps.append(Gap(code=GAP_TRUNCATED, reason=f"{what}: {', '.join(dict.fromkeys(changed))}")) + value = struct_pb2.Value() + json_format.ParseDict(safe, value) + return value + + +def json_safe(obj: Any) -> Any: + """``obj`` with JSON values only, as a ``google.protobuf.Struct`` or ``Value`` holds them.""" + return _json_safe(obj, 0, []) + + +def _json_safe(obj: Any, depth: int, changed: list[str]) -> Any: + if obj is None or isinstance(obj, (bool, str)): + return obj + if isinstance(obj, int): + if abs(obj) <= MAX_SAFE_INTEGER: + return obj + changed.append("integers beyond 2^53 as text") + return str(obj) + if isinstance(obj, float): + if math.isfinite(obj): + return obj + changed.append("non finite numbers set to null") + return None + if isinstance(obj, (bytes, bytearray)): + changed.append("bytes as base64") + return base64.b64encode(bytes(obj)).decode("ascii") + if depth >= MAX_VALUE_DEPTH: + changed.append(f"nesting cut at depth {MAX_VALUE_DEPTH}") + return None + if isinstance(obj, Mapping): + return {str(k): _json_safe(v, depth + 1, changed) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_json_safe(v, depth + 1, changed) for v in obj] + changed.append("values that are not JSON as text") + return str(obj) + + +def _str(value: Any) -> str: + return value if isinstance(value, str) else "" diff --git a/src/agentevals/builtin_metrics.py b/src/agentevals/builtin_metrics.py index 6f869ff2..cfe53832 100644 --- a/src/agentevals/builtin_metrics.py +++ b/src/agentevals/builtin_metrics.py @@ -5,6 +5,7 @@ import asyncio import inspect import logging +from dataclasses import dataclass from typing import Any from google.adk.evaluation.eval_case import ( @@ -55,6 +56,62 @@ } +_METRIC_CATEGORIES = { + "tool_trajectory_avg_score": "trajectory", + "response_match_score": "response", + "response_evaluation_score": "response", + "final_response_match_v2": "response", + "rubric_based_final_response_quality_v1": "quality", + "rubric_based_tool_use_quality_v1": "quality", + "hallucinations_v1": "safety", + "safety_v1": "safety", + "per_turn_user_simulator_quality_v1": "simulation", + "multi_turn_task_success_v1": "multi-turn", + "multi_turn_trajectory_quality_v1": "multi-turn", + "multi_turn_tool_use_quality_v1": "multi-turn", + "rubric_based_multi_turn_trajectory_quality_v1": "multi-turn", +} + + +@dataclass(frozen=True, slots=True) +class MetricDescription: + name: str + category: str + description: str + requires_eval_set: bool + requires_llm: bool + requires_gcp: bool + requires_rubrics: bool + working: bool + + +def metric_catalog() -> list[MetricDescription]: + """The built-in metrics agentevals can run, from ADK's registry.""" + from google.adk.evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY + + from .config import UNSUPPORTED_BUILTIN_METRICS + + out = [] + for metric in DEFAULT_METRIC_EVALUATOR_REGISTRY.get_registered_metrics(): + name = metric.metric_name + if name in UNSUPPORTED_BUILTIN_METRICS: + continue + rubrics = name.startswith("rubric_based_") + out.append( + MetricDescription( + name=name, + category=_METRIC_CATEGORIES.get(name, "other"), + description=metric.description or "No description available", + requires_eval_set=name in METRICS_NEEDING_EXPECTED, + requires_llm=name in METRICS_NEEDING_LLM, + requires_gcp=name in METRICS_NEEDING_GCP, + requires_rubrics=rubrics, + working=not rubrics, + ) + ) + return out + + def _to_invocation_events(inv: Invocation) -> Invocation: """Return a copy of *inv* with ``intermediate_data`` shaped as ``InvocationEvents``. diff --git a/src/agentevals/cli.py b/src/agentevals/cli.py index 448f14a3..a9a6e570 100644 --- a/src/agentevals/cli.py +++ b/src/agentevals/cli.py @@ -540,15 +540,18 @@ async def _run_servers( otlp_http_port: int, otlp_grpc_port: int, *, + api_grpc_port: int = 0, mcp_port: int | None = None, log_level: str = "warning", ) -> None: - """Start API, OTLP HTTP+gRPC receivers, and optional MCP (Streamable HTTP).""" + """Start the REST API and UI, the gRPC API, the OTLP HTTP and gRPC receivers, and optional MCP.""" import uvicorn from .api.app import create_app from .api.models import OtlpReceiverPorts from .api.otlp_app import create_otlp_app + from .api.rpc.server import ApiGrpcSettings + from .api.rpc.services import Ports from .streaming.manager import LiveManager shared_kwargs: dict = { @@ -563,6 +566,13 @@ async def _run_servers( trace_manager=mgr, enable_streaming=True, otlp_ports=OtlpReceiverPorts(http_port=otlp_http_port, grpc_port=otlp_grpc_port), + api_grpc=ApiGrpcSettings( + host=host, + port=api_grpc_port, + ports=Ports(http=port, otlp_http=otlp_http_port, otlp_grpc=otlp_grpc_port, api_grpc=api_grpc_port), + ) + if api_grpc_port + else None, ) otlp_app = create_otlp_app(trace_manager=mgr) @@ -629,6 +639,14 @@ async def _run_servers( default=4317, help="Port for OTLP gRPC receiver (default: 4317, standard OTLP gRPC port).", ) +@click.option( + "--api-grpc-port", + type=click.IntRange(min=0), + default=8002, + envvar="AGENTEVALS_API_GRPC_PORT", + show_envvar=True, + help="Port for the gRPC API (default: 8002; 0 turns it off). Expose it only to trusted callers.", +) @click.option( "--mcp-port", default=None, @@ -667,6 +685,7 @@ def serve( port: int, otlp_http_port: int, otlp_grpc_port: int, + api_grpc_port: int, mcp_port: int | None, eval_sets: str | None, headless: bool, @@ -717,46 +736,38 @@ def serve( click.echo(f" OTLP HTTP: http://{host}:{otlp_http_port} (OTEL_EXPORTER_OTLP_ENDPOINT default)") click.echo(f" OTLP gRPC: {host}:{otlp_grpc_port} (OTEL_EXPORTER_OTLP_PROTOCOL=grpc)") click.echo(f" API: http://{host}:{port}/api") - if mcp_port is not None: - click.echo(f" MCP: http://{host}:{mcp_port}/mcp") - click.echo(" Web UI: http://localhost:5173") - click.echo() - - if eval_sets: - click.echo(f" Eval sets: {eval_sets}") - click.echo() - - click.echo("Waiting for agent connections...") - click.echo() - - asyncio.run( - _run_servers( - host, - port, - otlp_http_port, - otlp_grpc_port, - mcp_port=mcp_port, - log_level="info", - ) - ) elif has_ui and not headless: click.echo(f"agentevals: http://{host}:{port}") click.echo(f" OTLP HTTP: http://{host}:{otlp_http_port}") click.echo(f" OTLP gRPC: {host}:{otlp_grpc_port}") - if mcp_port is not None: - click.echo(f" MCP (Streamable HTTP): http://{host}:{mcp_port}/mcp") - click.echo() - - asyncio.run(_run_servers(host, port, otlp_http_port, otlp_grpc_port, mcp_port=mcp_port)) else: click.echo(f"agentevals API: http://{host}:{port}/api") click.echo(f" OTLP HTTP: http://{host}:{otlp_http_port}") click.echo(f" OTLP gRPC: {host}:{otlp_grpc_port}") - if mcp_port is not None: - click.echo(f" MCP (Streamable HTTP): http://{host}:{mcp_port}/mcp") + if api_grpc_port: + click.echo(f" gRPC API: {host}:{api_grpc_port}") + if mcp_port is not None: + click.echo(f" MCP: http://{host}:{mcp_port}/mcp") + if dev: + click.echo(" Web UI: http://localhost:5173") + if eval_sets: + click.echo() + click.echo(f" Eval sets: {eval_sets}") click.echo() + click.echo("Waiting for agent connections...") + click.echo() - asyncio.run(_run_servers(host, port, otlp_http_port, otlp_grpc_port, mcp_port=mcp_port)) + asyncio.run( + _run_servers( + host, + port, + otlp_http_port, + otlp_grpc_port, + api_grpc_port=api_grpc_port, + mcp_port=mcp_port, + log_level="info" if dev else "warning", + ) + ) # --------------------------------------------------------------------------- diff --git a/src/agentevals/evaluation_events.py b/src/agentevals/evaluation_events.py index e2d90dbb..d692f3be 100644 --- a/src/agentevals/evaluation_events.py +++ b/src/agentevals/evaluation_events.py @@ -25,12 +25,15 @@ import os import threading import time -import uuid from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any from .otel.identity import EMITTER_SCOPE from .otel.model import Span, attr_str +from .telemetry import PROCESS_INSTANCE_ID, build_resource, otlp_protocol +from .telemetry import env_flag as _env_flag +from .telemetry import env_true as _env_true +from .telemetry import package_version as _package_version if TYPE_CHECKING: from .config import EvaluatorDef @@ -51,29 +54,6 @@ LABEL_NOT_EVALUATED = "not_evaluated" _LABELS = {"PASSED": LABEL_PASS, "FAILED": LABEL_FAIL, "NOT_EVALUATED": LABEL_NOT_EVALUATED} -_TRUE = {"1", "true", "yes", "on"} -_FALSE = {"0", "false", "no", "off"} - -_PROCESS_INSTANCE_ID = str(uuid.uuid4()) - - -def _env_true(name: str) -> bool: - """Lenient read for standard OTel switches, which other tools also set.""" - return os.environ.get(name, "").strip().lower() in _TRUE - - -def _env_flag(name: str) -> bool | None: - """Strict read for agentevals switches: ``None`` when unset, ``ValueError`` when unrecognized.""" - raw = os.environ.get(name) - if raw is None or not raw.strip(): - return None - value = raw.strip().lower() - if value in _TRUE: - return True - if value in _FALSE: - return False - raise ValueError(f"{name} must be one of true/false/1/0/yes/no/on/off (got: {raw!r})") - def disabled_by_otel() -> bool: """The standard switches that silence telemetry for the whole process.""" @@ -92,46 +72,14 @@ def enabled() -> bool: return _requested or bool(_env_flag(ENABLE_ENV)) -def _package_version() -> str: - from . import __version__ - - return __version__ - - -def build_resource() -> Any: - """Service identity as one unit; ``OTEL_RESOURCE_ATTRIBUTES`` and ``OTEL_SERVICE_NAME`` win.""" - from opentelemetry.sdk.resources import OTELResourceDetector, Resource - - ours = Resource.create( - { - "service.name": os.environ.get("OTEL_SERVICE_NAME") or "agentevals", - "service.version": _package_version(), - "service.instance.id": _PROCESS_INSTANCE_ID, - } - ) - return ours.merge(OTELResourceDetector().detect()) - - def instance_id() -> str: """The ``service.instance.id`` this process emits with (used by the live feedback guard).""" value = build_resource().attributes.get("service.instance.id") - return str(value) if value else _PROCESS_INSTANCE_ID - - -def _protocol() -> str: - raw = ( - os.environ.get("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL") or os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") or "" - ).strip() - if raw in ("", "http/protobuf"): - return "http/protobuf" - if raw == "grpc": - return "grpc" - logger.warning("OTLP protocol %r is not supported for evaluation events; using http/protobuf", raw) - return "http/protobuf" + return str(value) if value else PROCESS_INSTANCE_ID def _exporter_from_env() -> Any: - if _protocol() == "grpc": + if otlp_protocol("LOGS") == "grpc": from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter else: from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter diff --git a/src/agentevals/genai/extract.py b/src/agentevals/genai/extract.py index 40fa9767..caeb433b 100644 --- a/src/agentevals/genai/extract.py +++ b/src/agentevals/genai/extract.py @@ -1,6 +1,6 @@ """Turn extraction from GenAI telemetry. -Rules, in brief (``otel_native_core_202610.md`` sections 4.5 and 16): +Rules, in brief: * Spans are classified from the overlay's operation name, never from span names. * A turn is anchored on a workflow or agent span with no workflow, agent or tool ancestor. An @@ -57,7 +57,20 @@ tool_responses_of, user_turn_messages, ) -from .model import Conversation, ExternalEvaluation, LlmCall, ToolCall, Turn, Usage +from .model import ( + GAP_POSSIBLE_CONTINUATION, + GAP_TOOL_DATA_BY_NAME, + GAP_TOOL_FROM_PARTS_ONLY, + GAP_TOOL_NAME_MISSING, + GAP_TOOL_RESULT_BY_ORDER, + Conversation, + ExternalEvaluation, + Gap, + LlmCall, + ToolCall, + Turn, + Usage, +) from .overlay import SpanView, overlay INFERENCE = "inference" @@ -214,7 +227,7 @@ def _usage(view: SpanView) -> Usage | None: } if values["input_tokens"] is None and values["output_tokens"] is None: return None - return Usage(**{k: max(0, v or 0) for k, v in values.items()}) + return Usage(**{k: None if v is None else max(0, v) for k, v in values.items()}) def _status(span: Span) -> str: @@ -375,7 +388,7 @@ def _present(views: Sequence[SpanView], key: str) -> Any: return next((v.attrs[key] for v in views if v.attrs.get(key) is not None), None) -def _tool_calls_from_spans(analysis: TraceAnalysis, chains: list[list[str]], warnings: list[str]) -> list[ToolCall]: +def _tool_calls_from_spans(analysis: TraceAnalysis, chains: list[list[str]], warnings: list[Gap]) -> list[ToolCall]: out = [] for chain in chains: views = [analysis.info[s].view for s in chain] @@ -384,7 +397,12 @@ def _tool_calls_from_spans(analysis: TraceAnalysis, chains: list[list[str]], war if not name: tokens = span.name.split() name = tokens[1] if len(tokens) >= 2 and tokens[0] == sc.OP_EXECUTE_TOOL else span.name - warnings.append(f"tool span {chain[0]} has no gen_ai.tool.name; used span name token {name!r}") + warnings.append( + Gap( + code=GAP_TOOL_NAME_MISSING, + reason=f"tool span {chain[0]} has no gen_ai.tool.name; used span name token {name!r}", + ) + ) status = _first(views, lambda v: _status(v.span) if _status(v.span) != "unset" else None) error_type = _first(views, lambda v: attr_str(v.attrs, sc.ERROR_TYPE)) is_error = status == "error" or error_type is not None @@ -405,7 +423,7 @@ def _tool_calls_from_spans(analysis: TraceAnalysis, chains: list[list[str]], war return out -def _tool_calls_from_parts(calls: list[LlmCall], warnings: list[str] | None = None) -> list[ToolCall]: +def _tool_calls_from_parts(calls: list[LlmCall], warnings: list[Gap] | None = None) -> list[ToolCall]: """Tool calls from ``tool_call`` parts, results from later ``tool_call_response`` parts. A response pairs with its call by id. A response whose id matches no call in the turn (some @@ -436,8 +454,11 @@ def _tool_calls_from_parts(calls: list[LlmCall], warnings: list[str] | None = No ) if match is not None and responses[match][1]["id"] and warnings is not None: warnings.append( - f"tool {tc['name']} result joined by order" - f" (response id {responses[match][1]['id']!r} matches no call)" + Gap( + code=GAP_TOOL_RESULT_BY_ORDER, + reason=f"tool {tc['name']} result joined by order" + f" (response id {responses[match][1]['id']!r} matches no call)", + ) ) if match is not None: claimed.add(match) @@ -459,7 +480,7 @@ def _is_empty(value: Any) -> bool: return value is None or value == {} or value == "" or value == [] -def _fill_from_parts(span_tools: list[ToolCall], calls: list[LlmCall], warnings: list[str]) -> list[ToolCall]: +def _fill_from_parts(span_tools: list[ToolCall], calls: list[LlmCall], warnings: list[Gap]) -> list[ToolCall]: """Complete tool spans that carry no arguments or result from the turn's message parts, and return them together with the part tool calls no span claimed (a client side tool without a span, say). An unclaimed call goes right after the call requested before it, so the trajectory @@ -496,7 +517,11 @@ def _fill_from_parts(span_tools: list[ToolCall], calls: list[LlmCall], warnings: filled.append("result") if filled and by_name: warnings.append( - f"tool {tool.name} {' and '.join(filled)} joined from message parts by name (call ids differ)" + Gap( + code=GAP_TOOL_DATA_BY_NAME, + reason=f"tool {tool.name} {' and '.join(filled)} joined from message parts by name" + " (call ids differ)", + ) ) out = list(span_tools) previous: ToolCall | None = None @@ -504,7 +529,12 @@ def _fill_from_parts(span_tools: list[ToolCall], calls: list[LlmCall], warnings: if index in claimed: previous = claimed[index] continue - warnings.append(f"tool {part.name} appears only in message parts, without a tool span") + warnings.append( + Gap( + code=GAP_TOOL_FROM_PARTS_ONLY, + reason=f"tool {part.name} appears only in message parts, without a tool span", + ) + ) position = next(i for i, t in enumerate(out) if t is previous) + 1 if previous is not None else 0 out.insert(position, part) previous = part @@ -550,7 +580,7 @@ def _turn(analysis: TraceAnalysis, anchor: str) -> Turn: member_set = set(member_ids) ordered = [s.span_id for s in trace.walk(anchor) if s.span_id in member_set] anchor_view = info[anchor].view - warnings: list[str] = [] + warnings: list[Gap] = [] calls = _llm_calls(analysis, ordered) tool_chains = _tool_chains(analysis, ordered) @@ -581,7 +611,7 @@ def _turn(analysis: TraceAnalysis, anchor: str) -> Turn: usage = Usage() for call in calls: - if call.usage: + if call.usage is not None: usage = usage + call.usage agents: set[str] = set() @@ -729,9 +759,10 @@ def _join_continuations(turns: list[Turn]) -> list[Turn]: out[-1] = _merge(out[-1], turn) continue if out and _maybe_continues(out[-1], turn): - warning = ( - f"turn {turn.ref.span_id} may continue turn {out[-1].ref.span_id} after a tool call, " - "but message content was not captured; kept as its own turn" + warning = Gap( + code=GAP_POSSIBLE_CONTINUATION, + reason=f"turn {turn.ref.span_id} may continue turn {out[-1].ref.span_id} after a tool call, " + "but message content was not captured; kept as its own turn", ) turn = turn.model_copy(update={"warnings": [*turn.warnings, warning]}) out.append(turn) diff --git a/src/agentevals/genai/model.py b/src/agentevals/genai/model.py index 5e6dc565..f855dd39 100644 --- a/src/agentevals/genai/model.py +++ b/src/agentevals/genai/model.py @@ -17,14 +17,38 @@ class Usage(BaseModel): - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_input_tokens: int = 0 - cache_write_input_tokens: int = 0 - reasoning_output_tokens: int = 0 + """Token counts. ``None`` means not recorded, which is not the same as 0.""" + + input_tokens: int | None = None + output_tokens: int | None = None + cache_read_input_tokens: int | None = None + cache_write_input_tokens: int | None = None + reasoning_output_tokens: int | None = None def __add__(self, other: Usage) -> Usage: - return Usage(**{k: getattr(self, k) + getattr(other, k) for k in Usage.model_fields}) + return Usage(**{k: _add(getattr(self, k), getattr(other, k)) for k in Usage.model_fields}) + + +def _add(a: int | None, b: int | None) -> int | None: + if a is None: + return b + return a if b is None else a + b + + +GAP_TOOL_NAME_MISSING = "tool_name_missing" +GAP_TOOL_RESULT_BY_ORDER = "tool_result_by_order" +GAP_TOOL_DATA_BY_NAME = "tool_data_by_name" +GAP_TOOL_FROM_PARTS_ONLY = "tool_from_parts_only" +GAP_POSSIBLE_CONTINUATION = "possible_continuation" + + +class Gap(BaseModel): + """Why part of a turn is missing or approximate. ``code`` is stable; ``reason`` is for people.""" + + model_config = ConfigDict(frozen=True) + + code: str + reason: str class LlmCall(BaseModel): @@ -107,7 +131,7 @@ class Turn(BaseModel): conversation_id: str | None = None framework_ids: dict[str, str] = Field(default_factory=dict) external_evaluations: list[ExternalEvaluation] = Field(default_factory=list) - warnings: list[str] = Field(default_factory=list) + warnings: list[Gap] = Field(default_factory=list) class Conversation(BaseModel): diff --git a/src/agentevals/genai/routing.py b/src/agentevals/genai/routing.py index f0421b6a..2294613d 100644 --- a/src/agentevals/genai/routing.py +++ b/src/agentevals/genai/routing.py @@ -11,8 +11,8 @@ from types import MappingProxyType from typing import Any -from ..otel.identity import EMITTER_SCOPE -from ..otel.model import LogRecord, Span +from ..otel.identity import API_SPAN_PREFIX, EMITTER_SCOPE, GRPC_INSTRUMENTATION_SCOPE +from ..otel.model import SPAN_KIND_SERVER, LogRecord, Span from .extract import IGNORED, NON_GENAI, classify from .grouping import KeyKind, identity_key from .overlay import overlay @@ -60,3 +60,12 @@ def is_own_record(self, log: LogRecord) -> bool: return True instance = log.resource.attributes.get(SERVICE_INSTANCE_ID) return self.emitter_instance_id is not None and instance == self.emitter_instance_id + + def is_own_span(self, span: Span) -> bool: + """A server span of agentevals' API (``agentevals.telemetry``). Matched on the instrumentation + scope and the method, not on ``service.instance.id``, which a producer may share.""" + return ( + span.kind == SPAN_KIND_SERVER + and span.scope.name == GRPC_INSTRUMENTATION_SCOPE + and span.name.startswith(API_SPAN_PREFIX) + ) diff --git a/src/agentevals/loader/__init__.py b/src/agentevals/loader/__init__.py index ffc55e7c..b688c37e 100644 --- a/src/agentevals/loader/__init__.py +++ b/src/agentevals/loader/__init__.py @@ -15,6 +15,7 @@ detect_format, get_loader_for_format, load_telemetry, + load_telemetry_text, load_traces, load_traces_from_obj, ) @@ -24,6 +25,7 @@ __all__ = [ "TelemetryFile", "load_telemetry", + "load_telemetry_text", "JAEGER_JSON", "OTLP_JSON", "JaegerJsonLoader", diff --git a/src/agentevals/loader/auto.py b/src/agentevals/loader/auto.py index bcef3d12..6b057b30 100644 --- a/src/agentevals/loader/auto.py +++ b/src/agentevals/loader/auto.py @@ -98,26 +98,31 @@ def load_telemetry(path: str, *, format: str | None = None) -> TelemetryFile: """Read a trace file into envelopes. Raises ``ValueError`` when the content is not recognized.""" with open(path, encoding="utf-8") as f: content = f.read() + return load_telemetry_text(content, format=format, name=path) + + +def load_telemetry_text(content: str, *, format: str | None = None, name: str = "document") -> TelemetryFile: + """Read trace file content into envelopes. ``name`` only appears in messages.""" detected, data = _classify(content) fmt = format or detected if fmt not in _LOADERS: if format is not None: raise ValueError(f"Unknown trace format {format!r}. Supported: {sorted(_LOADERS)}") raise ValueError( - f"Could not detect trace format for {path!r}. " + f"Could not detect trace format for {name!r}. " 'Expected Jaeger JSON ({"data": [...]}) or OTLP JSON ' '({"resourceSpans": [...]} / {"batches": [...]}). ' "Pass an explicit format= override if needed." ) if data is _UNPARSED: if fmt == JAEGER_JSON: - raise ValueError(f"Invalid Jaeger JSON in {path!r}") + raise ValueError(f"Invalid Jaeger JSON in {name!r}") result = decode_otlp_text(content) else: result = _decode_obj(data, fmt) traces, unattributed = build_traces(result.spans, result.logs) if result.reasons: - logger.warning("Skipped items in %s: %s", path, result.error_message) + logger.warning("Skipped items in %s: %s", name, result.error_message) return TelemetryFile(format=fmt, traces=traces, unattributed_logs=unattributed, warnings=result.warnings) diff --git a/src/agentevals/otel/decode.py b/src/agentevals/otel/decode.py index 32c558c0..d6ea3f3c 100644 --- a/src/agentevals/otel/decode.py +++ b/src/agentevals/otel/decode.py @@ -5,7 +5,7 @@ * OTLP/JSON documents (dicts), including legacy and vendor variants: protobuf field names (``resource_spans``, ``trace_id``), ``batches`` and ``instrumentationLibrarySpans`` (Tempo v1), the ``{"trace": {...}}`` wrapper (Tempo v2), - attributes given as flat or nested dicts (ClickHouse JSON columns), enum names instead of + attributes given as flat or nested dicts (JSON column exports), enum names instead of numbers, and bare spans with no resource envelope (one span per line). * Protobuf ``Export{Trace,Logs}ServiceRequest`` messages, read field by field. ``MessageToDict`` is never used: it base64 encodes ids and costs a full extra copy. diff --git a/src/agentevals/otel/identity.py b/src/agentevals/otel/identity.py index 04538476..f1d1999e 100644 --- a/src/agentevals/otel/identity.py +++ b/src/agentevals/otel/identity.py @@ -16,6 +16,10 @@ # Instrumentation scope of the evaluation result records agentevals emits. EMITTER_SCOPE = "agentevals" +# agentevals' own API server spans: the upstream gRPC instrumentation's scope, and the span name +# prefix of agentevals' methods. +GRPC_INSTRUMENTATION_SCOPE = "opentelemetry.instrumentation.grpc" +API_SPAN_PREFIX = "/agentevals.api." MAX_KEY_LENGTH = 256 diff --git a/src/agentevals/otel/store.py b/src/agentevals/otel/store.py index bca681d1..97ebb955 100644 --- a/src/agentevals/otel/store.py +++ b/src/agentevals/otel/store.py @@ -6,8 +6,8 @@ only records what happened in :attr:`TelemetryStore.events`. What the telemetry means is the :class:`RoutingPolicy`'s business (``genai.routing`` for GenAI); -this module holds only the mechanics. Routing (``otel_native_core_202610.md`` 16.2), for spans of -trace T with session key K from the policy: +this module holds only the mechanics. Routing, for spans of trace T with session key K from the +policy: * R1, membership first: a trace already in a session stays there, with two exceptions. A session that learns a stronger key from its own trace (any key, for a provisional session) is promoted @@ -70,7 +70,12 @@ def is_own_record(self, log: LogRecord) -> bool: """A record agentevals emitted itself, refused so a pipeline loop cannot feed it back.""" ... + def is_own_span(self, span: Span) -> bool: + """A span of agentevals' own server, accepted and dropped so a pipeline loop cannot store it.""" + ... + +DROPPED_OWN_SPANS = "span emitted by agentevals" REASON_TRACE_SPANS = "span(s) rejected: trace span limit reached" REASON_SESSION_SPANS = "span(s) rejected: session span limit reached" REASON_SESSION_TRACES = "span(s) rejected: session trace limit reached" @@ -207,7 +212,6 @@ class LiveSession: log_count: int = 0 nbytes: int = 0 last_activity: float = 0.0 - invocations: list[dict[str, Any]] = field(default_factory=list) @property def provisional(self) -> bool: @@ -566,6 +570,12 @@ def _dirty(self, session: LiveSession) -> None: def ingest_spans(self, spans: Sequence[Span]) -> IngestResult: """Apply spans of one trace sharing one resource (one routing unit).""" result = IngestResult() + own = sum(1 for s in spans if self.policy.is_own_span(s)) + if own: + result.accepted += own + result.dropped[DROPPED_OWN_SPANS] += own + self.counters[DROPPED_OWN_SPANS] += own + spans = [s for s in spans if not self.policy.is_own_span(s)] if not spans: return result now = self.clock() @@ -673,7 +683,6 @@ def ingest_spans(self, spans: Sequence[Span]) -> IngestResult: trace.complete = False session.last_activity = now self._schedule_trace(trace) - self.events.append(("spans", session.session_id, added)) self._dirty(session) return result diff --git a/src/agentevals/resolvers/kubernetes.py b/src/agentevals/resolvers/kubernetes.py index 9c2d4003..7d3d33d9 100644 --- a/src/agentevals/resolvers/kubernetes.py +++ b/src/agentevals/resolvers/kubernetes.py @@ -34,8 +34,7 @@ async def resolve(self, ref: dict[str, Any]) -> str: secret = await asyncio.to_thread(self._core_v1.read_namespaced_secret, name, namespace) data = secret.data or {} if key not in data: - available = ", ".join(sorted(data)) or "(none)" - raise ValueError(f"key '{key}' not found in Secret {namespace}/{name}; available keys: {available}") + raise ValueError(f"key '{key}' not found in Secret {namespace}/{name}") return base64.b64decode(data[key]).decode("utf-8") diff --git a/src/agentevals/run/fetcher.py b/src/agentevals/run/fetcher.py index b7058ca8..17741ada 100644 --- a/src/agentevals/run/fetcher.py +++ b/src/agentevals/run/fetcher.py @@ -9,7 +9,9 @@ from __future__ import annotations import logging +import re from typing import Protocol +from urllib.parse import quote import httpx @@ -19,6 +21,15 @@ logger = logging.getLogger(__name__) +_TRACE_ID = re.compile(r"[0-9a-fA-F]{16}(?:[0-9a-fA-F]{16})?") + + +def validate_trace_id(trace_id: str) -> str: + """A 16 or 32 character hex trace id. Anything else could change the fetched URL's path.""" + if not _TRACE_ID.fullmatch(trace_id): + raise ValueError("target.trace_id must be 16 or 32 hex characters") + return trace_id + class TraceFetcher(Protocol): async def fetch(self, target: TraceTarget, context: dict) -> list[Trace]: ... @@ -42,7 +53,7 @@ def __init__(self, timeout_s: float = 30.0) -> None: async def fetch(self, target: TraceTarget, context: dict) -> list[Trace]: if not target.base_url or not target.trace_id: raise ValueError("HttpTraceFetcher requires target.base_url and target.trace_id") - url = target.base_url.rstrip("/") + "/" + target.trace_id + url = target.base_url.rstrip("/") + "/" + quote(validate_trace_id(target.trace_id), safe="") headers = (context.get("headers") if isinstance(context, dict) else {}) or {} async with httpx.AsyncClient(timeout=self._timeout_s) as client: resp = await client.get(url, headers=headers) diff --git a/src/agentevals/run/service.py b/src/agentevals/run/service.py index 0f98bf27..a7fe3c7f 100644 --- a/src/agentevals/run/service.py +++ b/src/agentevals/run/service.py @@ -20,6 +20,7 @@ from ..runner import RunResult from ..storage.models import Run, RunSpec, RunStatus, TraceTarget from ..storage.repos import ResultRepository, RunRepository +from .fetcher import validate_trace_id from .result_builder import build_results, summarize_run_result logger = logging.getLogger(__name__) @@ -60,6 +61,8 @@ def _validate_spec(spec: RunSpec) -> None: EvalParams.model_validate(spec.eval_config or {}) except ValidationError as exc: raise ValueError(f"Invalid eval_config: {exc}") from exc + if spec.target.kind == "http": + validate_trace_id(spec.target.trace_id or "") async def get(self, run_id: UUID) -> Run | None: return await self._runs.get(run_id) @@ -107,7 +110,7 @@ async def record_eval_run( spec = RunSpec( approach="trace_replay", target=target, - eval_config=params.model_dump(by_alias=False), + eval_config=params.model_dump(by_alias=False, include=set(EvalParams.model_fields)), eval_set=eval_set_dict, ) diff --git a/src/agentevals/run/sinks.py b/src/agentevals/run/sinks.py index db228b30..c8c4e9bd 100644 --- a/src/agentevals/run/sinks.py +++ b/src/agentevals/run/sinks.py @@ -24,6 +24,7 @@ from importlib.metadata import entry_points from pathlib import Path from typing import Any, Protocol, cast +from urllib.parse import urlsplit from uuid import UUID import httpx @@ -91,6 +92,12 @@ async def emit_error(self, run_id: UUID, error: str, attempt: int) -> None: await self._write({"phase": "error", "run_id": str(run_id), "error": error}) +def _origin(url: str) -> str: + """Scheme and host for logs: webhook URLs often carry their token in the path or query.""" + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc.rpartition('@')[2]}" if parts.scheme and parts.netloc else "(url)" + + class HttpWebhookSink: """POST JSON to a URL with retries. @@ -134,14 +141,13 @@ async def _post(self, payload: dict) -> None: if resp.status_code < 500: if resp.status_code >= 400: logger.warning( - "Webhook %s returned %d: %s (run_id=%s)", - self._url, + "Webhook %s returned %d (run_id=%s)", + _origin(self._url), resp.status_code, - resp.text[:200], payload.get("run_id"), ) return - last_exc = RuntimeError(f"HTTP {resp.status_code}: {resp.text[:200]}") + last_exc = RuntimeError(f"HTTP {resp.status_code}") except (httpx.HTTPError, RuntimeError) as exc: last_exc = exc if attempt < self._max_attempts: @@ -149,9 +155,9 @@ async def _post(self, payload: dict) -> None: delay = min(delay * 2, 10.0) logger.error( "Webhook %s failed after %d attempts: %s (run_id=%s)", - self._url, + _origin(self._url), self._max_attempts, - last_exc, + type(last_exc).__name__ if isinstance(last_exc, httpx.HTTPError) else last_exc, payload.get("run_id"), ) diff --git a/src/agentevals/runner.py b/src/agentevals/runner.py index a91bd297..dee61092 100644 --- a/src/agentevals/runner.py +++ b/src/agentevals/runner.py @@ -260,7 +260,7 @@ async def _evaluate_group( trace_result.conversation_id = conversation.conversation_id trace_result.turn_refs = [TurnRef(trace_id=t.ref.trace_id, span_id=t.ref.span_id) for t in conversation.turns] trace_result.conversion_warnings = sorted( - {w for trace in traces for w in trace.warnings} | {w for t in conversation.turns for w in t.warnings} + {w for trace in traces for w in trace.warnings} | {w.reason for t in conversation.turns for w in t.warnings} ) if not conversation.turns: diff --git a/src/agentevals/storage/postgres/migrations/000001_init.up.sql b/src/agentevals/storage/postgres/migrations/000001_init.up.sql index 98bcabaa..49676c6f 100644 --- a/src/agentevals/storage/postgres/migrations/000001_init.up.sql +++ b/src/agentevals/storage/postgres/migrations/000001_init.up.sql @@ -25,8 +25,8 @@ CREATE INDEX IF NOT EXISTS session_expires_at_idx WHERE expires_at IS NOT NULL; -- Reserved for future per-span / per-log persistence. Spans and logs stay --- in-process on StreamingTraceManager in this OSS slice; this table exists --- so a future migration can populate it without an ALTER on session. +-- in process in the live manager; this table exists so a future migration +-- can populate it without an ALTER on session. CREATE TABLE IF NOT EXISTS {schema}.session_event ( event_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, session_id TEXT NOT NULL REFERENCES {schema}.session(session_id) ON DELETE CASCADE, @@ -97,7 +97,7 @@ CREATE INDEX IF NOT EXISTS result_run_id_idx ON {schema}.result (run_id); CREATE INDEX IF NOT EXISTS result_expires_at_idx ON {schema}.result (expires_at) WHERE expires_at IS NOT NULL; -- Reserved for cached evaluator code from external sources (GitHub today, --- additional sources later). No read/write code in this slice; included here +-- additional sources later). Nothing reads or writes it yet; included here -- so a future change does not require an ALTER on this table. CREATE TABLE IF NOT EXISTS {schema}.evaluator_cache ( source_name TEXT NOT NULL, diff --git a/src/agentevals/storage/repos/__init__.py b/src/agentevals/storage/repos/__init__.py index c267be27..21d8d05c 100644 --- a/src/agentevals/storage/repos/__init__.py +++ b/src/agentevals/storage/repos/__init__.py @@ -16,8 +16,8 @@ class SessionRepository(Protocol): """Tracks streaming TraceSession metadata. - Spans and logs themselves stay in-process on the StreamingTraceManager in - this OSS slice; only the session lifecycle row is persisted. + Spans and logs themselves stay in process in the live manager; only the + session lifecycle row is persisted. """ async def get(self, session_id: str) -> "TraceSession | None": ... diff --git a/src/agentevals/storage/repos/postgres.py b/src/agentevals/storage/repos/postgres.py index c27467ef..32d4b11a 100644 --- a/src/agentevals/storage/repos/postgres.py +++ b/src/agentevals/storage/repos/postgres.py @@ -230,6 +230,21 @@ async def claim_next(self, *, worker_id: str, lease: timedelta, max_attempts: in lease_seconds = int(lease.total_seconds()) async with self._pool.acquire() as conn: async with conn.transaction(): + # A run whose lease expired on its last attempt, or after cancel was requested, is + # never claimed again; finish it so it does not stay running forever. + await conn.execute( + f""" + UPDATE {self._t} + SET status = CASE WHEN cancel_requested THEN 'cancelled' ELSE 'failed' END, + error = CASE WHEN cancel_requested THEN error + ELSE 'worker lease expired on the last attempt' END, + finished_at = now() + WHERE status = 'running' + AND lease_expires_at < now() + AND (attempt >= $1 OR cancel_requested) + """, + max_attempts, + ) row = await conn.fetchrow( f""" UPDATE {self._t} @@ -341,7 +356,7 @@ async def upsert_many(self, run_id: UUID, results: list[Result]) -> None: r.evaluator_type, r.status.value, r.score, - [s for s in r.per_invocation_scores if s is not None], + list(r.per_invocation_scores), r.trace_id, r.span_id, json.dumps(r.details or {}), diff --git a/src/agentevals/streaming/manager.py b/src/agentevals/streaming/manager.py index dfd801e2..fd1f04da 100644 --- a/src/agentevals/streaming/manager.py +++ b/src/agentevals/streaming/manager.py @@ -1,11 +1,13 @@ -"""Live session manager: ingest into the store, completion ticker, recompute and the SSE diff. +"""Live session manager: ingest into the store, completion ticker, recompute and the watchers. The store is synchronous; this module owns the event loop side. Exports are applied one routing unit at a time, yielding the loop on a time budget so a large export never holds it. Conversations -are recomputed on frozen snapshots in a worker thread, with turns memoized per trace version, and -the result is diffed against what each session already sent. The diff only appends: every live -element has an identity, so a span arriving late never repeats or retracts an element. After a -session has completed once, only ``session_complete`` is sent for it. +are recomputed on frozen snapshots in a worker thread, with turns memoized per trace version. + +Each recompute builds the session's view (``api.views``) in the same thread and publishes it to +every watcher: the gRPC ``WatchSessions`` stream and the UI's SSE stream. A watcher keeps only the +latest pending event per session, so it never holds more than the session limit and a slow reader +is never dropped: it skips intermediate states. """ from __future__ import annotations @@ -19,41 +21,35 @@ import time from collections import OrderedDict from collections.abc import Iterable, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any -from ..api.models import ( - SessionInfo, - WSSessionCompleteEvent, - WSSessionRemovedEvent, - WSSessionStartedEvent, -) +from google.protobuf import json_format + +from ..api.models import SessionInfo +from ..api.v1alpha1 import sessions_pb2 +from ..api.views import SessionFacts, session_view from ..genai.extract import conversation_from_turns, extract_turns from ..genai.grouping import SESSION_NAME -from ..genai.messages import has_text, text_of, user_turn_messages -from ..genai.model import Conversation, ToolCall, Turn +from ..genai.model import Conversation, Turn from ..genai.routing import GenAIRoutingPolicy from ..otel.decode import DecodeResult from ..otel.encode import encode_traces -from ..otel.model import LogRecord, Span, SpanRef, Trace, attr_float, attr_int, build_traces +from ..otel.model import LogRecord, Span, Trace, build_traces from ..otel.store import IngestResult, Limits, LiveSession, TelemetryStore, TraceSnapshot logger = logging.getLogger(__name__) YIELD_INTERVAL_SECONDS = 0.01 -SSE_QUEUE_SIZE = 1_000 -MAX_EMITTED_KEYS = 10_000 DEBOUNCE_MIN_SECONDS = 0.25 DEBOUNCE_MAX_SECONDS = 5.0 DEBOUNCE_MAX_WAIT_SECONDS = 1.0 EXPIRE_INTERVAL_SECONDS = 10.0 MAX_TICK_SECONDS = 1.0 -# A recompute can produce thousands of elements at once; handing the loop back between batches -# lets SSE writers drain, so the queue bound drops only clients that are really behind. -BROADCAST_BATCH = 100 - MAX_BYTES_ENV = "AGENTEVALS_LIVE_MAX_BYTES" +MAX_WATCHERS = 32 +_SYNCED_KEY = "" def _max_bytes_from_env() -> int | None: @@ -74,198 +70,72 @@ def _resource_groups(items: Iterable[Any]) -> list[list[Any]]: return list(groups.values()) -def _span_stub(span: Span) -> dict[str, Any]: - return {"traceId": span.trace_id, "spanId": span.span_id, "parentSpanId": span.parent_span_id, "name": span.name} - - -def model_info(turn: Turn, spans: dict[tuple[str, str], Span]) -> dict[str, Any]: - """Per turn model metadata; usage counts each logical call once.""" - info: dict[str, Any] = {} - calls = turn.llm_calls - usage = turn.usage - leaf = spans.get((calls[0].ref.trace_id, calls[0].ref.span_id)) if calls else None - for key, value in ( - ("models", sorted({c.request_model for c in calls if c.request_model})), - ("inputTokens", usage.input_tokens), - ("outputTokens", usage.output_tokens), - ("provider", next((c.provider for c in calls if c.provider), None)), - ("responseModels", sorted({c.response_model for c in calls if c.response_model})), - ("finishReasons", sorted({r for c in calls for r in c.finish_reasons})), - ("cacheCreationTokens", usage.cache_write_input_tokens), - ("cacheReadTokens", usage.cache_read_input_tokens), - ("temperature", attr_float(leaf.attributes, "gen_ai.request.temperature") if leaf else None), - ("maxTokens", attr_int(leaf.attributes, "gen_ai.request.max_tokens") if leaf else None), - ("errorTypes", sorted({c.error_type for c in calls if c.error_type})), - ): - if value: - info[key] = value - return info - - -def invocation_dict(turn: Turn, spans: dict[tuple[str, str], Span]) -> dict[str, Any]: - """A turn as the ``session_complete`` invocation the live UI renders.""" - return { - "invocationId": turn.ref.span_id, - "traceId": turn.ref.trace_id, - "userText": text_of(turn.user_input) or "", - "agentText": text_of(turn.final_output) or "", - "toolCalls": [{"name": t.name, "args": t.arguments or {}, "id": t.call_id} for t in turn.tool_calls], - "toolResponses": [ - {"name": t.name, "response": t.result, "id": t.call_id} for t in turn.tool_calls if t.result is not None - ], - "modelInfo": model_info(turn, spans), - "status": turn.status, - "errorType": turn.error_type, - "contentCaptured": turn.content_captured, - "warnings": turn.warnings, - "externalEvaluations": [ - { - "name": e.name, - "scoreValue": e.score_value, - "scoreLabel": e.score_label, - "explanation": e.explanation, - "spanId": e.ref.span_id, - "sourceService": e.source_service, - } - for e in turn.external_evaluations - ], - } +@dataclass(frozen=True, slots=True) +class WatchEvent: + """One ``WatchSessionsResponse`` in the forms watchers send. Shared and never mutated.""" + with_turns: sessions_pb2.WatchSessionsResponse + summary_only: sessions_pb2.WatchSessionsResponse + json: str -@dataclass(slots=True) -class LiveState: - """What one session already sent over SSE.""" - - keys: set = field(default_factory=set) - input_tokens: int = 0 - output_tokens: int = 0 - - def seen(self, key: Any) -> bool: - return key in self.keys - - def mark(self, *keys: Any) -> bool: - """Record keys; False when the cap is reached (the element is then left to ``session_complete``).""" - if len(self.keys) + len(keys) > MAX_EMITTED_KEYS: - return False - self.keys.update(keys) - return True - - -def _ref_key(kind: str, ref: SpanRef | None) -> tuple | None: - return (kind, ref.trace_id, ref.span_id) if ref else None - - -def _tool_idents(tool: ToolCall) -> list[str]: - """Every identity a tool call has had so far: its call id, then each of its spans. - - The layers of one call (framework tool span, MCP client, MCP server) can arrive in separate - exports, so the call's outermost span, and with it ``ref``, changes as they come in. - """ - idents = [tool.call_id] if tool.call_id else [] - idents += [r.span_id for r in (tool.ref, *tool.inner_refs) if r is not None] - return idents or [f"{tool.name}@{tool.start_ns}"] - - -def _first_sighting(state: LiveState, keys: list) -> bool: - """True the first time any of ``keys`` is seen. Later sightings record the keys they add, so - an element that gains an identity is still recognized under it.""" - if any(state.seen(k) for k in keys): - state.mark(*(k for k in keys if not state.seen(k))) - return False - return state.mark(*keys) - - -def live_events(session_id: str, conversation: Conversation, state: LiveState, incomplete: set[str]) -> list[dict]: - """Elements of ``conversation`` not sent yet, as the SSE events the live UI consumes.""" - events: list[dict] = [] - last_invocation = None - for turn in conversation.turns: - invocation = turn.ref.trace_id if turn.ref.trace_id in incomplete else turn.ref.span_id - last_invocation = invocation - base = {"sessionId": session_id, "invocationId": invocation} - - text = text_of(turn.user_input) - if text: - source = next((c for c in turn.llm_calls if user_turn_messages(c.input_messages)), None) - keys = [("user-text", turn.ref.trace_id, text), _ref_key("user", turn.ref)] - if source is not None: - keys += [_ref_key("user", source.ref), _ref_key("user", source.wrapper_ref)] - keys = [k for k in keys if k] - if not any(state.seen(k) for k in keys) and state.mark(*keys): - events.append({"type": "user_input", **base, "text": text, "timestamp": turn.start_ns / 1e9}) - - spoke = False - for call in turn.llm_calls: - if call.delegated or not has_text(call.output_messages): - continue - spoke = True - keys = [k for k in (_ref_key("agent", call.ref), _ref_key("agent", call.wrapper_ref)) if k] - if any(state.seen(k) for k in keys) or not state.mark(*keys): - continue - events.append( - { - "type": "agent_response", - **base, - "text": text_of(call.output_messages) or "", - "timestamp": call.end_ns / 1e9, - } - ) - final = text_of(turn.final_output) - if not spoke and final: - key = _ref_key("agent-final", turn.ref) - if not state.seen(key) and state.mark(key): - events.append({"type": "agent_response", **base, "text": final, "timestamp": turn.end_ns / 1e9}) - - for tool in turn.tool_calls: - idents = _tool_idents(tool) - ident = idents[0] - if _first_sighting(state, [("tool", i) for i in idents]): - events.append( - { - "type": "tool_call", - **base, - "toolCall": {"id": ident, "name": tool.name, "args": tool.arguments or {}}, - "timestamp": (tool.start_ns or turn.start_ns) / 1e9, - } - ) - if tool.result is not None and _first_sighting(state, [("result", i) for i in idents]): - events.append( - { - "type": "tool_result", - **base, - "toolCallId": tool.call_id or ident, - "toolName": tool.name, - "response": tool.result, - "isError": tool.is_error, - "timestamp": (tool.start_ns or turn.end_ns) / 1e9, - } - ) - - total_in = sum(t.usage.input_tokens for t in conversation.turns) - total_out = sum(t.usage.output_tokens for t in conversation.turns) - if total_in != state.input_tokens or total_out != state.output_tokens: - model = next( - (c.response_model or c.request_model for t in reversed(conversation.turns) for c in reversed(t.llm_calls)), - None, - ) - events.append( - { - "type": "token_update", - "sessionId": session_id, - "invocationId": last_invocation, - "inputTokens": total_in - state.input_tokens, - "outputTokens": total_out - state.output_tokens, - "model": model, - } - ) - state.input_tokens, state.output_tokens = total_in, total_out - return events + @classmethod + def of(cls, with_turns: sessions_pb2.WatchSessionsResponse, summary_only=None) -> WatchEvent: + return cls(with_turns, summary_only or with_turns, json_format.MessageToJson(with_turns, indent=None)) + + @classmethod + def upsert(cls, session) -> WatchEvent: + summary_only = sessions_pb2.WatchSessionsResponse() + summary_only.upsert.summary.CopyFrom(session.summary) + return cls.of(sessions_pb2.WatchSessionsResponse(upsert=session), summary_only) + + +class TooManyWatchers(RuntimeError): + pass + + +class Watcher: + """One subscriber: the latest pending event per session, in the order they changed.""" + def __init__(self, include_turns: bool) -> None: + self.include_turns = include_turns + self._pending: OrderedDict[str, WatchEvent] = OrderedDict() + self._wake = asyncio.Event() + self.closed = False -class SseClient: - def __init__(self) -> None: - self.queue: asyncio.Queue = asyncio.Queue(maxsize=SSE_QUEUE_SIZE) - self.dropped = False + def put(self, key: str, event: WatchEvent) -> None: + self._pending.pop(key, None) + self._pending[key] = event + self._wake.set() + + def close(self) -> None: + self.closed = True + self._wake.set() + + async def next(self) -> WatchEvent | None: + """The next event, or ``None`` once the watcher is closed.""" + while not self._pending: + if self.closed: + return None + self._wake.clear() + await self._wake.wait() + return self._pending.popitem(last=False)[1] + + def response(self, event: WatchEvent) -> sessions_pb2.WatchSessionsResponse: + return event.with_turns if self.include_turns else event.summary_only + + +def session_facts(session: LiveSession) -> SessionFacts: + return SessionFacts( + session_id=session.session_id, + received_at=session.started_at, + is_complete=session.is_complete, + trace_ids=tuple(session.trace_ids), + span_count=session.span_count, + eval_set_id=session.eval_set_id, + ) + + +_SYNCED = WatchEvent.of(sessions_pb2.WatchSessionsResponse(synced=sessions_pb2.Synced())) @dataclass(slots=True) @@ -302,8 +172,8 @@ def __init__( overrides["max_bytes"] = budget limits = dataclasses.replace(limits or Limits(), **overrides) self.store = TelemetryStore(GenAIRoutingPolicy(emitter_instance_id), limits) - self.clients: list[SseClient] = [] - self._states: dict[str, LiveState] = {} + self.watchers: list[Watcher] = [] + self._views: dict[str, WatchEvent] = {} self._pending: dict[str, _Pending] = {} self._running: dict[str, asyncio.Task] = {} self._durations: dict[str, float] = {} @@ -325,8 +195,7 @@ def start(self) -> None: self._ticker = asyncio.create_task(self._run()) async def shutdown(self) -> None: - for client in list(self.clients): - self._close(client) + self.close_watchers() tasks = [t for t in (self._ticker, *self._running.values()) if t is not None] self._ticker = None for task in tasks: @@ -339,37 +208,44 @@ def _poke(self) -> None: if self._wake is not None: self._wake.set() - # ------------------------------------------------------------ SSE - - def register_sse_client(self) -> SseClient: - client = SseClient() - self.clients.append(client) - return client - - def unregister_sse_client(self, client: SseClient) -> None: - if client in self.clients: - self.clients.remove(client) - - def _close(self, client: SseClient) -> None: - self.unregister_sse_client(client) - while not client.queue.empty(): - client.queue.get_nowait() - client.queue.put_nowait(None) - - def broadcast(self, event: dict) -> None: - """Queue an event for every client. A client whose queue is full is dropped; the UI - reconnects and refetches the session list.""" - for client in list(self.clients): - try: - client.queue.put_nowait(event) - except asyncio.QueueFull: - client.dropped = True - logger.warning("SSE client fell behind by %d events; disconnecting it", SSE_QUEUE_SIZE) - self._close(client) + # ------------------------------------------------------------ watchers + + def subscribe(self, *, include_turns: bool) -> Watcher: + """A watcher holding every current session, newest first, then ``synced``.""" + if len(self.watchers) >= MAX_WATCHERS: + raise TooManyWatchers(f"at most {MAX_WATCHERS} session watchers") + watcher = Watcher(include_turns) + for session in sorted(self.store.sessions.values(), key=lambda s: s.started_at, reverse=True): + watcher.put(session.session_id, self.view(session)) + watcher.put(_SYNCED_KEY, _SYNCED) + self.watchers.append(watcher) + return watcher + + def close_watchers(self) -> None: + for watcher in list(self.watchers): + self.unsubscribe(watcher) + + def unsubscribe(self, watcher: Watcher) -> None: + if watcher in self.watchers: + self.watchers.remove(watcher) + watcher.close() + + def view(self, session: LiveSession) -> WatchEvent: + """The session's latest view; a summary from the store until its first recompute.""" + cached = self._views.get(session.session_id) + return cached if cached is not None else WatchEvent.upsert(session_view(session_facts(session))) + + def _publish(self, session_id: str, event: WatchEvent) -> None: + for watcher in self.watchers: + watcher.put(session_id, event) + + def _publish_removed(self, session_id: str, absorbed_by: str | None) -> None: + removed = sessions_pb2.SessionRemoved(id=session_id, absorbed_by=absorbed_by or "") + self._publish(session_id, WatchEvent.of(sessions_pb2.WatchSessionsResponse(removed=removed))) # ------------------------------------------------------------ views - def session_info(self, session: LiveSession, *, with_invocations: bool = False) -> SessionInfo: + def session_info(self, session: LiveSession) -> SessionInfo: return SessionInfo( session_id=session.session_id, trace_id=self.store.primary_trace_id(session), @@ -378,7 +254,6 @@ def session_info(self, session: LiveSession, *, with_invocations: bool = False) is_complete=session.is_complete, started_at=session.started_at.isoformat(), metadata=session.metadata, - invocations=session.invocations if with_invocations else None, ) def session_traces(self, session: LiveSession | str) -> list[Trace]: @@ -438,16 +313,10 @@ def _drain(self) -> None: if kind == "started": session = self.store.sessions.get(session_id) if session is not None: - self._states[session_id] = LiveState() - self.broadcast(WSSessionStartedEvent(session=self.session_info(session)).model_dump(by_alias=True)) + self._publish(session_id, self.view(session)) elif kind == "removed": self._forget(session_id) - self.broadcast( - WSSessionRemovedEvent(session_id=session_id, absorbed_by=event[2]).model_dump(by_alias=True) - ) - elif kind == "spans": - stubs = [_span_stub(s) for s in event[2]] - self.broadcast({"type": "span_received", "sessionId": session_id, "span": stubs[0], "spans": stubs}) + self._publish_removed(session_id, event[2]) elif kind == "dirty": pending = self._pending.get(session_id) if pending is None: @@ -458,7 +327,7 @@ def _drain(self) -> None: logger.info("Session %s reopened by new spans", session_id) def _forget(self, session_id: str) -> None: - self._states.pop(session_id, None) + self._views.pop(session_id, None) self._pending.pop(session_id, None) self._durations.pop(session_id, None) task = self._running.pop(session_id, None) @@ -523,34 +392,28 @@ def tick(self) -> None: async def _recompute(self, session_id: str, completed_now: bool) -> None: try: + live = self.store.sessions.get(session_id) + if live is None: + return + facts = session_facts(live) snapshots = self.store.snapshot(session_id) started = time.monotonic() - conversation, spans = await asyncio.to_thread(self._conversation, session_id, snapshots) - invocations = await asyncio.to_thread(lambda: [invocation_dict(t, spans) for t in conversation.turns]) + view, turns = await asyncio.to_thread(self._view, session_id, facts, snapshots) self._durations[session_id] = time.monotonic() - started - session = self.store.sessions.get(session_id) - if session is None: + if self.store.sessions.get(session_id) is not live: return - session.invocations = invocations - if not session.completed_once: - state = self._states.setdefault(session_id, LiveState()) - incomplete = {s.trace_id for s in snapshots if not s.complete} - for n, event in enumerate(live_events(session_id, conversation, state, incomplete), start=1): - self.broadcast(event) - if n % BROADCAST_BATCH == 0: - await asyncio.sleep(0) - if session.is_complete and (completed_now or session.completed_once): - session.completed_once = True - logger.info( - "Session complete: %s (%d spans, %d logs, %d turns)", - session_id, - session.span_count, - session.log_count, - len(invocations), - ) - self.broadcast( - WSSessionCompleteEvent(session_id=session_id, invocations=invocations).model_dump(by_alias=True) - ) + self._views[session_id] = view + self._publish(session_id, view) + if live.is_complete and (completed_now or live.completed_once): + if not live.completed_once: + logger.info( + "Session complete: %s (%d spans, %d logs, %d turns)", + session_id, + live.span_count, + live.log_count, + turns, + ) + live.completed_once = True except asyncio.CancelledError: raise except Exception: @@ -561,6 +424,11 @@ async def _recompute(self, session_id: str, completed_now: bool) -> None: if session_id in self._pending: self._poke() + def _view(self, session_id: str, facts: SessionFacts, snapshots: list[TraceSnapshot]) -> tuple[WatchEvent, int]: + """The session's view and turn count. Runs in a worker thread, on copied facts and snapshots.""" + conversation, spans = self._conversation(session_id, snapshots) + return WatchEvent.upsert(session_view(facts, conversation, spans)), len(conversation.turns) + def _conversation( self, session_id: str, snapshots: list[TraceSnapshot] ) -> tuple[Conversation, dict[tuple[str, str], Span]]: diff --git a/src/agentevals/telemetry.py b/src/agentevals/telemetry.py new file mode 100644 index 00000000..ecb21c2a --- /dev/null +++ b/src/agentevals/telemetry.py @@ -0,0 +1,110 @@ +"""agentevals' own telemetry: identity, the env switches, and the opt in API server traces. + +agentevals is a telemetry sink, so it exports nothing of its own unless asked to. Each signal it +emits has a dedicated provider, never the global one, so an application that embeds agentevals +keeps its own providers. Exporters read only the standard ``OTEL_EXPORTER_OTLP_*`` variables. + +API server traces come from the upstream gRPC instrumentation. They are on with +``AGENTEVALS_API_TRACES=true``; ``OTEL_SDK_DISABLED=true`` or ``OTEL_TRACES_EXPORTER=none`` win. +``OTEL_TRACES_EXPORTER=console`` prints them instead of exporting. The live store drops these spans +(``genai.routing``), so pointing the exporter at agentevals itself does not create sessions. +""" + +from __future__ import annotations + +import logging +import os +import uuid +from typing import Any + +logger = logging.getLogger(__name__) + +API_TRACES_ENV = "AGENTEVALS_API_TRACES" + +PROCESS_INSTANCE_ID = str(uuid.uuid4()) + +_TRUE = {"1", "true", "yes", "on"} +_FALSE = {"0", "false", "no", "off"} + + +def env_true(name: str) -> bool: + """Lenient read for standard OTel switches, which other tools also set.""" + return os.environ.get(name, "").strip().lower() in _TRUE + + +def env_flag(name: str) -> bool | None: + """Strict read for agentevals switches: ``None`` when unset, ``ValueError`` when unrecognized.""" + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return None + value = raw.strip().lower() + if value in _TRUE: + return True + if value in _FALSE: + return False + raise ValueError(f"{name} must be one of true/false/1/0/yes/no/on/off (got: {raw!r})") + + +def package_version() -> str: + from . import __version__ + + return __version__ + + +def build_resource() -> Any: + """Service identity as one unit; ``OTEL_RESOURCE_ATTRIBUTES`` and ``OTEL_SERVICE_NAME`` win.""" + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + + ours = Resource.create( + { + "service.name": os.environ.get("OTEL_SERVICE_NAME") or "agentevals", + "service.version": package_version(), + "service.instance.id": PROCESS_INSTANCE_ID, + } + ) + return ours.merge(OTELResourceDetector().detect()) + + +def otlp_protocol(signal: str) -> str: + """``http/protobuf`` or ``grpc``, from ``OTEL_EXPORTER_OTLP__PROTOCOL``, then the general one.""" + raw = ( + os.environ.get(f"OTEL_EXPORTER_OTLP_{signal}_PROTOCOL") or os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") or "" + ).strip() + if raw in ("", "http/protobuf"): + return "http/protobuf" + if raw == "grpc": + return "grpc" + logger.warning("OTLP protocol %r is not supported for %s; using http/protobuf", raw, signal.lower()) + return "http/protobuf" + + +def _traces_exporter_setting() -> str: + return os.environ.get("OTEL_TRACES_EXPORTER", "").strip().lower() + + +def api_traces_enabled() -> bool: + if env_true("OTEL_SDK_DISABLED") or _traces_exporter_setting() == "none": + return False + return bool(env_flag(API_TRACES_ENV)) + + +def api_tracer_provider() -> Any | None: + """A dedicated provider for the API server's spans, or ``None`` when they are off.""" + if not api_traces_enabled(): + return None + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + + if _traces_exporter_setting() == "console": + exporter = ConsoleSpanExporter() + elif otlp_protocol("TRACES") == "grpc": + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + + exporter = OTLPSpanExporter() + else: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + + exporter = OTLPSpanExporter() + provider = TracerProvider(resource=build_resource()) + provider.add_span_processor(BatchSpanProcessor(exporter)) + return provider diff --git a/src/agentevals/trace_metrics.py b/src/agentevals/trace_metrics.py index 634dd658..39b957cf 100644 --- a/src/agentevals/trace_metrics.py +++ b/src/agentevals/trace_metrics.py @@ -97,7 +97,7 @@ def extract_performance_metrics(conversation: Conversation, traces: Sequence[Tra ] with_usage = [c.usage for c in calls if c.usage is not None] - per_call_totals = [u.input_tokens + u.output_tokens for u in with_usage] + per_call_totals = [(u.input_tokens or 0) + (u.output_tokens or 0) for u in with_usage] models = sorted({c.request_model or c.response_model for c in calls if c.request_model or c.response_model}) if not turn_latencies: @@ -110,12 +110,12 @@ def extract_performance_metrics(conversation: Conversation, traces: Sequence[Tra "tool_executions": _calc_summary_stats(tool_latencies), }, "tokens": { - "total_prompt": sum(u.input_tokens for u in with_usage), - "total_output": sum(u.output_tokens for u in with_usage), + "total_prompt": sum(u.input_tokens or 0 for u in with_usage), + "total_output": sum(u.output_tokens or 0 for u in with_usage), "total": sum(per_call_totals), "per_llm_call": _calc_summary_stats([float(x) for x in per_call_totals]), - "cache_creation_tokens": sum(u.cache_write_input_tokens for u in with_usage), - "cache_read_tokens": sum(u.cache_read_input_tokens for u in with_usage), + "cache_creation_tokens": sum(u.cache_write_input_tokens or 0 for u in with_usage), + "cache_read_tokens": sum(u.cache_read_input_tokens or 0 for u in with_usage), }, "counts": { "llm_calls": len(calls), diff --git a/tests/api/test_api_tracing.py b/tests/api/test_api_tracing.py new file mode 100644 index 00000000..d3391e1e --- /dev/null +++ b/tests/api/test_api_tracing.py @@ -0,0 +1,95 @@ +"""Opt in traces of the API server, through the upstream gRPC instrumentation.""" + +from __future__ import annotations + +import grpc +import pytest +from grpc_health.v1 import health_pb2, health_pb2_grpc +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind + +from agentevals.api.otlp_processing import decode_traces_protobuf, ingest_traces +from agentevals.api.rpc.server import start_api_server, stop_api_server +from agentevals.api.rpc.services import ApiContext +from agentevals.api.v1alpha1 import system_pb2, system_pb2_grpc +from agentevals.otel.identity import GRPC_INSTRUMENTATION_SCOPE +from agentevals.otel.store import DROPPED_OWN_SPANS +from agentevals.streaming.manager import LiveManager +from agentevals.telemetry import api_tracer_provider, api_traces_enabled + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for name in ("AGENTEVALS_API_TRACES", "OTEL_SDK_DISABLED", "OTEL_TRACES_EXPORTER", "AGENTEVALS_GRPC_REFLECTION"): + monkeypatch.delenv(name, raising=False) + + +class TestSwitches: + @pytest.mark.parametrize( + ("env", "enabled"), + [ + ({}, False), + ({"AGENTEVALS_API_TRACES": "true"}, True), + ({"AGENTEVALS_API_TRACES": "false"}, False), + ({"AGENTEVALS_API_TRACES": "true", "OTEL_SDK_DISABLED": "true"}, False), + ({"AGENTEVALS_API_TRACES": "true", "OTEL_TRACES_EXPORTER": "none"}, False), + ({"AGENTEVALS_API_TRACES": "true", "OTEL_TRACES_EXPORTER": "console"}, True), + ], + ) + def test_enabled(self, monkeypatch, env, enabled): + for name, value in env.items(): + monkeypatch.setenv(name, value) + assert api_traces_enabled() is enabled + + def test_an_unrecognized_value_is_an_error(self, monkeypatch): + monkeypatch.setenv("AGENTEVALS_API_TRACES", "sometimes") + with pytest.raises(ValueError, match="AGENTEVALS_API_TRACES"): + api_traces_enabled() + + def test_no_provider_when_off(self): + assert api_tracer_provider() is None + + def test_a_dedicated_provider_when_on(self, monkeypatch): + from opentelemetry import trace + + monkeypatch.setenv("AGENTEVALS_API_TRACES", "true") + monkeypatch.setenv("OTEL_TRACES_EXPORTER", "console") + provider = api_tracer_provider() + try: + assert isinstance(provider, TracerProvider) + assert provider is not trace.get_tracer_provider() + assert provider.resource.attributes["service.name"] == "agentevals" + finally: + provider.shutdown() + + +async def test_server_spans_are_recorded_and_never_become_sessions(monkeypatch): + exporter = InMemorySpanExporter() + # A session name on the resource would open a session for any span the store kept. + provider = TracerProvider(resource=Resource.create({"agentevals.session_name": "api-self"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr("agentevals.api.rpc.server.api_tracer_provider", lambda: provider) + + manager = LiveManager() + api = await start_api_server(ApiContext(manager=manager), "127.0.0.1", 0) + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{api.port}") as channel: + await system_pb2_grpc.SystemServiceStub(channel).GetVersion(system_pb2.GetVersionRequest()) + await health_pb2_grpc.HealthStub(channel).Check(health_pb2.HealthCheckRequest()) + finally: + await stop_api_server(api) + + spans = exporter.get_finished_spans() + assert [s.name for s in spans] == ["/agentevals.api.v1alpha1.SystemService/GetVersion"] + [own] = spans + assert own.kind == SpanKind.SERVER + assert own.instrumentation_scope.name == GRPC_INSTRUMENTATION_SCOPE + + result = await ingest_traces(decode_traces_protobuf(encode_spans(spans).SerializeToString()), manager) + assert (result.accepted, result.rejected) == (1, 0) + assert manager.store.counters[DROPPED_OWN_SPANS] == 1 + assert manager.sessions == {} diff --git a/tests/api/test_evaluate_persistence.py b/tests/api/test_evaluate_persistence.py index 1a456dcc..df203e2a 100644 --- a/tests/api/test_evaluate_persistence.py +++ b/tests/api/test_evaluate_persistence.py @@ -1,4 +1,4 @@ -"""Option A: /api/evaluate variants persist when run_service is configured. +"""/api/evaluate variants persist when run_service is configured. These tests stub a memory-backed RunService onto app.state so we can drive the persistence path without standing up a real Postgres. The lifespan @@ -11,12 +11,14 @@ import asyncio import json +import tempfile from pathlib import Path import pytest from fastapi.testclient import TestClient from agentevals.api.app import create_app +from agentevals.config import EvalParams from agentevals.run.service import RunService from agentevals.storage.repos.memory import MemoryRepos @@ -80,6 +82,20 @@ def test_run_persisted_when_run_service_set(self, app_with_runs): assert run.spec.target.trace_files == ["helm.json"] assert run.spec.target.trace_count == 1 + def test_recorded_run_keeps_no_server_paths(self, app_with_runs): + app, repos = app_with_runs + with TestClient(app) as client: + with SAMPLE_TRACE.open("rb") as f: + r = client.post( + "/api/evaluate", + files={"trace_files": ("helm.json", f, "application/json")}, + data={"config": '{"evaluators": [{"name": "tool_trajectory_avg_score", "type": "builtin"}]}'}, + ) + assert r.status_code == 200 + [run] = asyncio.run(repos.runs.list()) + assert set(run.spec.eval_config) <= set(EvalParams.model_fields) + assert tempfile.gettempdir() not in run.spec.model_dump_json() + def test_results_persisted_alongside_run(self, app_with_runs): app, repos = app_with_runs with TestClient(app) as client: @@ -98,7 +114,7 @@ def test_results_persisted_alongside_run(self, app_with_runs): def test_each_call_creates_distinct_run(self, app_with_runs): """Multiple UI uploads accumulate in run history; each gets its own - Run row. This is the core OSS user value of Option A.""" + Run row.""" app, repos = app_with_runs with TestClient(app) as client: for _ in range(3): diff --git a/tests/api/test_grpc_api.py b/tests/api/test_grpc_api.py new file mode 100644 index 00000000..c623a503 --- /dev/null +++ b/tests/api/test_grpc_api.py @@ -0,0 +1,741 @@ +"""The v1alpha1 gRPC API, served in process on an ephemeral port and called through real stubs.""" + +from __future__ import annotations + +import asyncio +import base64 +import copy +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from unittest.mock import AsyncMock, patch +from uuid import UUID, uuid4 + +import grpc +import pytest +from fastapi.testclient import TestClient +from google.protobuf import json_format, struct_pb2 +from google.rpc import error_details_pb2, status_pb2 +from grpc_health.v1 import health_pb2, health_pb2_grpc +from grpc_reflection.v1alpha import reflection_pb2, reflection_pb2_grpc +from opentelemetry.proto.trace.v1 import trace_pb2 + +from agentevals import __version__ +from agentevals.api.app import create_app +from agentevals.api.rpc import convert, services +from agentevals.api.rpc.server import start_api_server, stop_api_server +from agentevals.api.rpc.services import ApiContext, Ports, SystemServicer +from agentevals.api.v1alpha1 import ( + common_pb2, + conversion_pb2, + conversion_pb2_grpc, + evaluation_pb2, + evaluation_pb2_grpc, + runs_pb2, + runs_pb2_grpc, + sessions_pb2, + sessions_pb2_grpc, + system_pb2, + system_pb2_grpc, + views_pb2, +) +from agentevals.builtin_metrics import metric_catalog +from agentevals.otel.decode import decode_json_document +from agentevals.run.service import RunService +from agentevals.storage.models import Result, ResultStatus, Run, RunSpec, RunStatus, TraceTarget +from agentevals.storage.repos.memory import MemoryRepos +from agentevals.streaming import manager as manager_module +from agentevals.streaming.manager import LiveManager + +FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "otlp" / "hand" / "adk_go_main.json" +DOCUMENT = json.loads(FIXTURE.read_text()) +TRACE_ID = "1dbd779ff94bf24c7b84678a5dc0346c" +PORTS = Ports(http=8001, otlp_http=4318, otlp_grpc=4317, api_grpc=8002) +# Not a hex string, so it cannot turn up inside a generated id by chance. +MARK = "qxmark" +SERVICES = [ + "agentevals.api.v1alpha1.SessionService", + "agentevals.api.v1alpha1.EvaluationService", + "agentevals.api.v1alpha1.RunService", + "agentevals.api.v1alpha1.ConversionService", + "agentevals.api.v1alpha1.SystemService", +] + + +@pytest.fixture +def repos(): + return MemoryRepos.create() + + +@pytest.fixture +def context(repos, monkeypatch): + for name in ("AGENTEVALS_API_TRACES", "AGENTEVALS_GRPC_REFLECTION"): + monkeypatch.delenv(name, raising=False) + return ApiContext(manager=LiveManager(), run_service=RunService(repos.runs, repos.results), ports=PORTS) + + +@pytest.fixture +async def channel(context): + api = await start_api_server(context, "127.0.0.1", 0) + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{api.port}") as channel: + yield channel + finally: + await stop_api_server(api) + + +def _document(content: bytes | None = None, name: str = "adk_go_main.json") -> common_pb2.TraceDocument: + return common_pb2.TraceDocument( + content=FIXTURE.read_bytes() if content is None else content, + format=common_pb2.TRACE_FORMAT_OTLP_JSON, + name=name, + ) + + +def _traces_data(document: dict) -> trace_pb2.TracesData: + """The OTLP/JSON document as protobuf. OTLP/JSON writes ids as hex, the protobuf JSON mapping as base64.""" + data = {"resourceSpans": copy.deepcopy(document["resourceSpans"])} + for resource_spans in data["resourceSpans"]: + for scope_spans in resource_spans["scopeSpans"]: + for span in scope_spans["spans"]: + for key in ("traceId", "spanId", "parentSpanId"): + if span.get(key): + span[key] = base64.b64encode(bytes.fromhex(span[key])).decode() + return json_format.ParseDict(data, trace_pb2.TracesData()) + + +def _error_info(error: grpc.aio.AioRpcError) -> error_details_pb2.ErrorInfo | None: + for key, value in error.trailing_metadata() or (): + if key == "grpc-status-details-bin": + for detail in status_pb2.Status.FromString(value).details: + info = error_details_pb2.ErrorInfo() + if detail.Unpack(info): + return info + return None + + +def _trailers(error: grpc.aio.AioRpcError) -> bytes: + return b"".join(v if isinstance(v, bytes) else v.encode() for _, v in error.trailing_metadata() or ()) + + +async def _load(manager: LiveManager, session_id: str, document: dict = DOCUMENT) -> None: + decoded = decode_json_document(document, strict=True) + manager.load_session(session_id, decoded.spans, decoded.logs) + for _ in range(100): + manager.tick() + if not manager._running and not manager._pending: + return + await asyncio.sleep(0.01) + raise AssertionError("session recompute did not finish") + + +async def _read_until_synced(call) -> list[sessions_pb2.WatchSessionsResponse]: + messages = [] + while True: + message = await asyncio.wait_for(call.read(), 5) + messages.append(message) + if message.HasField("synced"): + return messages + + +async def _stream(call) -> list: + return [message async for message in call] + + +async def _read_to_end(call) -> list: + messages = [] + while (message := await call.read()) is not grpc.aio.EOF: + messages.append(message) + return messages + + +async def _golden(channel) -> struct_pb2.Struct: + response = await conversion_pb2_grpc.ConversionServiceStub(channel).ConvertTraces( + conversion_pb2.ConvertTracesRequest(traces=common_pb2.TraceInput(documents=[_document()])) + ) + return response.conversations[0].golden_adk_eval_set + + +def _trajectory_params() -> evaluation_pb2.EvalParams: + return evaluation_pb2.EvalParams( + evaluators=[ + evaluation_pb2.Evaluator(builtin=evaluation_pb2.BuiltinEvaluator(name="tool_trajectory_avg_score")) + ], + max_concurrent_evals=1, + ) + + +def _run_spec(**fields) -> runs_pb2.RunSpec: + return runs_pb2.RunSpec( + target=runs_pb2.TraceTarget(inline=_document()), + eval_config=_trajectory_params(), + **fields, + ) + + +class TestHealthAndReflection: + async def test_every_service_reports_serving(self, channel): + stub = health_pb2_grpc.HealthStub(channel) + for name in ["", *SERVICES]: + response = await stub.Check(health_pb2.HealthCheckRequest(service=name)) + assert response.status == health_pb2.HealthCheckResponse.SERVING, name + + async def test_reflection_is_off_by_default(self, channel): + call = reflection_pb2_grpc.ServerReflectionStub(channel).ServerReflectionInfo( + iter([reflection_pb2.ServerReflectionRequest(list_services="")]) + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(call) + assert error.value.code() == grpc.StatusCode.UNIMPLEMENTED + + async def test_reflection_lists_the_services_when_enabled(self, context, monkeypatch): + monkeypatch.setenv("AGENTEVALS_GRPC_REFLECTION", "true") + api = await start_api_server(context, "127.0.0.1", 0) + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{api.port}") as channel: + call = reflection_pb2_grpc.ServerReflectionStub(channel).ServerReflectionInfo( + iter([reflection_pb2.ServerReflectionRequest(list_services="")]) + ) + [response] = await _stream(call) + finally: + await stop_api_server(api) + names = {s.name for s in response.list_services_response.service} + assert set(SERVICES) <= names + + +class TestPolicies: + async def test_a_method_without_a_policy_refuses_startup(self, context, monkeypatch): + class Unchecked(SystemServicer): + async def GetVersion(self, request, context): # noqa: N802 + return system_pb2.GetVersionResponse() + + real = services.servicers + + def with_unchecked(ctx): + return [ + (Unchecked(ctx), register, service) + if isinstance(servicer, SystemServicer) + else (servicer, register, service) + for servicer, register, service in real(ctx) + ] + + monkeypatch.setattr("agentevals.api.rpc.server.servicers", with_unchecked) + with pytest.raises(RuntimeError, match="SystemService methods without an rpc policy: GetVersion"): + await start_api_server(context, "127.0.0.1", 0) + + async def test_unexpected_errors_are_masked(self, channel, caplog): + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), params=_trajectory_params() + ) + failing = AsyncMock(side_effect=RuntimeError(f"connection to {MARK}.internal refused")) + with patch("agentevals.api.evaluation.run_evaluation_from_traces", failing): + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + assert error.value.code() == grpc.StatusCode.INTERNAL + assert error.value.details() == "internal server error" + assert _error_info(error.value) is None + assert MARK.encode() not in _trailers(error.value) + assert MARK in caplog.text + + async def test_invalid_parameters_name_the_field_but_not_the_value(self, channel): + params = evaluation_pb2.EvalParams( + evaluators=[evaluation_pb2.Evaluator(code=evaluation_pb2.CodeEvaluator(name="c", path=f"/srv/{MARK}.exe"))] + ) + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), params=params + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert "evaluators.0" in error.value.details() + assert MARK not in error.value.details() + assert MARK.encode() not in _trailers(error.value) + assert _error_info(error.value).reason == "INVALID_REQUEST" + + +class TestSessions: + async def test_list_pages_newest_first(self, channel, context): + for name in ("first", "second", "third"): + await _load(context.manager, name) + stub = sessions_pb2_grpc.SessionServiceStub(channel) + page = await stub.ListSessions(sessions_pb2.ListSessionsRequest(page=common_pb2.PageRequest(limit=2))) + assert [s.id for s in page.sessions] == ["third", "second"] + rest = await stub.ListSessions( + sessions_pb2.ListSessionsRequest(page=common_pb2.PageRequest(limit=2, page_token=page.page.next_page_token)) + ) + assert [s.id for s in rest.sessions] == ["first"] + assert rest.page.next_page_token == "" + + @pytest.mark.parametrize("page", [common_pb2.PageRequest(limit=101), common_pb2.PageRequest(page_token="%%")]) + async def test_bad_paging_is_invalid_argument(self, channel, page): + with pytest.raises(grpc.aio.AioRpcError) as error: + await sessions_pb2_grpc.SessionServiceStub(channel).ListSessions( + sessions_pb2.ListSessionsRequest(page=page) + ) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert _error_info(error.value).domain == "agentevals.api.v1alpha1" + + async def test_get_session_returns_the_turns(self, channel, context): + await _load(context.manager, "weather") + response = await sessions_pb2_grpc.SessionServiceStub(channel).GetSession( + sessions_pb2.GetSessionRequest(id="weather") + ) + summary, [turn] = response.session.summary, response.session.turns + assert summary.state == views_pb2.SESSION_STATE_COMPLETE + assert (summary.totals.usage.input_tokens, summary.totals.usage.output_tokens) == (90, 18) + assert not summary.totals.usage.HasField("cache_read_input_tokens") + assert turn.user_text == "What is the weather in Paris?" + assert [t.name for t in turn.tool_calls] == ["get_weather"] + assert json_format.MessageToDict(turn.tool_calls[0].arguments) == {"city": "Paris"} + + async def test_get_unknown_session_is_not_found(self, channel): + with pytest.raises(grpc.aio.AioRpcError) as error: + await sessions_pb2_grpc.SessionServiceStub(channel).GetSession(sessions_pb2.GetSessionRequest(id="nope")) + assert error.value.code() == grpc.StatusCode.NOT_FOUND + + async def test_watch_sends_the_snapshot_then_synced_then_changes(self, channel, context): + await _load(context.manager, "before") + call = sessions_pb2_grpc.SessionServiceStub(channel).WatchSessions( + sessions_pb2.WatchSessionsRequest(include_turns=True) + ) + snapshot = await _read_until_synced(call) + assert [m.upsert.summary.id for m in snapshot[:-1]] == ["before"] + assert snapshot[0].upsert.turns[0].user_text == "What is the weather in Paris?" + await _load(context.manager, "after") + change = await asyncio.wait_for(call.read(), 5) + assert change.upsert.summary.id == "after" + call.cancel() + + async def test_watch_without_turns_sends_summaries(self, channel, context): + await _load(context.manager, "weather") + call = sessions_pb2_grpc.SessionServiceStub(channel).WatchSessions(sessions_pb2.WatchSessionsRequest()) + upsert = (await _read_until_synced(call))[0].upsert + call.cancel() + assert (len(upsert.turns), upsert.summary.totals.turns) == (0, 1) + + async def test_ui_stream_frames_equal_the_watch_messages(self, channel, context, monkeypatch): + await _load(context.manager, "weather") + call = sessions_pb2_grpc.SessionServiceStub(channel).WatchSessions( + sessions_pb2.WatchSessionsRequest(include_turns=True) + ) + watched = await _read_until_synced(call) + call.cancel() + + mgr = context.manager + subscribe = mgr.subscribe + + def subscribe_closed(**kwargs): + # A closed watcher still hands out what it holds, so the stream ends after the snapshot. + watcher = subscribe(**kwargs) + watcher.close() + return watcher + + monkeypatch.setattr(mgr, "subscribe", subscribe_closed) + body = TestClient(create_app(enable_streaming=True, trace_manager=mgr)).get("/stream/ui-updates").text + frames = [line.removeprefix("data: ") for line in body.splitlines() if line.startswith("data: ")] + assert [json_format.Parse(f, sessions_pb2.WatchSessionsResponse()) for f in frames] == watched + + async def test_watchers_are_capped_and_released(self, channel, context, monkeypatch): + monkeypatch.setattr(manager_module, "MAX_WATCHERS", 1) + stub = sessions_pb2_grpc.SessionServiceStub(channel) + call = stub.WatchSessions(sessions_pb2.WatchSessionsRequest()) + await _read_until_synced(call) + + with pytest.raises(grpc.aio.AioRpcError) as error: + await _read_until_synced(stub.WatchSessions(sessions_pb2.WatchSessionsRequest())) + assert error.value.code() == grpc.StatusCode.RESOURCE_EXHAUSTED + assert ( + TestClient(create_app(enable_streaming=True, trace_manager=context.manager)) + .get("/stream/ui-updates") + .status_code + == 503 + ) + + call.cancel() + for _ in range(100): + if not context.manager.watchers: + break + await asyncio.sleep(0.01) + assert context.manager.watchers == [] + + +class TestEvaluation: + async def test_list_metrics_is_the_catalog(self, channel): + response = await evaluation_pb2_grpc.EvaluationServiceStub(channel).ListMetrics( + evaluation_pb2.ListMetricsRequest() + ) + assert [m.name for m in response.metrics] == [m.name for m in metric_catalog()] + + async def test_evaluate_traces_streams_until_done(self, channel): + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), + adk_eval_set=await _golden(channel), + params=_trajectory_params(), + ) + events = await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + kinds = [e.WhichOneof("event") for e in events] + assert kinds[0] == "performance" and kinds[-1] == "done" + assert kinds.index("trace_result") < kinds.index("done") + [metric] = events[-1].done.trace_results[0].metric_results + assert (metric.metric_name, metric.score, metric.eval_status) == ( + "tool_trajectory_avg_score", + 1.0, + evaluation_pb2.EVAL_STATUS_PASSED, + ) + assert events[0].performance.trace_id == TRACE_ID + + async def test_events_match_the_rest_stream(self, channel): + golden = await _golden(channel) + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), adk_eval_set=golden, params=_trajectory_params() + ) + events = await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + + body = ( + TestClient(create_app()) + .post( + "/api/evaluate/json/stream", + json={ + "traces": DOCUMENT, + "evalSet": convert.from_struct(golden), + "config": { + "evaluators": [{"name": "tool_trajectory_avg_score", "type": "builtin"}], + "maxConcurrentEvals": 1, + }, + }, + ) + .text + ) + frames = [] + for block in body.strip().split("\n\n"): + lines = block.splitlines() + data = json.loads(lines[-1].removeprefix("data: ")) + if lines[0] == "event: performance_metrics": + frames.append(("performance", data)) + elif "message" in data: + frames.append(("progress", data)) + elif "traceProgress" in data: + frames.append(("trace_result", data)) + else: + frames.append(("done", data)) + + assert [e.WhichOneof("event") for e in events] == [kind for kind, _ in frames] + assert [e.progress.message for e in events if e.HasField("progress")] == [ + data["message"] for kind, data in frames if kind == "progress" + ] + rest_metric = frames[-1][1]["result"]["traceResults"][0]["metricResults"][0] + grpc_metric = events[-1].done.trace_results[0].metric_results[0] + assert (rest_metric["metricName"], rest_metric["score"]) == (grpc_metric.metric_name, grpc_metric.score) + + async def test_unresolvable_credential_is_failed_precondition(self, channel): + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), + params=_trajectory_params(), + credential_refs={"judge": convert.to_struct({"kind": f"{MARK}-resolver", "name": MARK})}, + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + assert error.value.code() == grpc.StatusCode.FAILED_PRECONDITION + assert MARK not in error.value.details() + + async def test_evaluator_without_a_kind_is_invalid_argument(self, channel): + request = evaluation_pb2.EvaluateTracesRequest( + traces=common_pb2.TraceInput(documents=[_document()]), + params=evaluation_pb2.EvalParams(evaluators=[evaluation_pb2.Evaluator()]), + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(evaluation_pb2_grpc.EvaluationServiceStub(channel).EvaluateTraces(request)) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + + +class TestRuns: + @pytest.mark.parametrize( + ("method", "request_message"), + [ + ("SubmitRun", runs_pb2.SubmitRunRequest()), + ("GetRun", runs_pb2.GetRunRequest(run_id=str(uuid4()))), + ("ListRuns", runs_pb2.ListRunsRequest()), + ("CancelRun", runs_pb2.CancelRunRequest(run_id=str(uuid4()))), + ("ListRunResults", runs_pb2.ListRunResultsRequest(run_id=str(uuid4()))), + ("WatchRun", runs_pb2.WatchRunRequest(run_id=str(uuid4()))), + ], + ) + async def test_runs_need_storage(self, channel, context, method, request_message): + context.run_service = None + call = getattr(runs_pb2_grpc.RunServiceStub(channel), method)(request_message) + with pytest.raises(grpc.aio.AioRpcError) as error: + await (_stream(call) if method == "WatchRun" else call) + assert error.value.code() == grpc.StatusCode.FAILED_PRECONDITION + info = _error_info(error.value) + assert (info.reason, info.domain) == ("STORAGE_NOT_CONFIGURED", "agentevals.api.v1alpha1") + + async def test_submit_then_get(self, channel): + stub = runs_pb2_grpc.RunServiceStub(channel) + submitted = (await stub.SubmitRun(runs_pb2.SubmitRunRequest(spec=_run_spec()))).run + assert submitted.status == runs_pb2.RUN_STATUS_QUEUED + fetched = (await stub.GetRun(runs_pb2.GetRunRequest(run_id=submitted.run_id))).run + assert fetched == submitted + assert json.loads(fetched.spec.target.inline.content) == DOCUMENT + assert fetched.spec.eval_config.evaluators[0].builtin.name == "tool_trajectory_avg_score" + + async def test_an_otlp_target_is_stored_as_otlp_json(self, channel, repos): + spec = runs_pb2.RunSpec( + target=runs_pb2.TraceTarget(otlp=common_pb2.OtlpTelemetry(traces=_traces_data(DOCUMENT))), + eval_config=_trajectory_params(), + ) + run = (await runs_pb2_grpc.RunServiceStub(channel).SubmitRun(runs_pb2.SubmitRunRequest(spec=spec))).run + stored = await repos.runs.get(UUID(run.run_id)) + assert stored.spec.target.trace_format == "otlp-json" + spans = [s for rs in stored.spec.target.inline["resourceSpans"] for ss in rs["scopeSpans"] for s in ss["spans"]] + assert {s["traceId"] for s in spans} == {TRACE_ID} + assert len(spans) == sum(len(ss["spans"]) for rs in DOCUMENT["resourceSpans"] for ss in rs["scopeSpans"]) + + async def test_reads_never_return_secrets(self, channel, repos): + spec = runs_pb2.RunSpec( + target=runs_pb2.TraceTarget( + http=runs_pb2.HttpTraceTarget( + base_url=f"https://reader:{MARK}@traces.example.com/api?key={MARK}", trace_id=TRACE_ID + ) + ), + eval_config=_trajectory_params(), + context=runs_pb2.RunContext(headers={"Authorization": f"Bearer {MARK}"}), + sinks=[ + convert.to_struct( + { + "kind": "http_webhook", + "url": f"https://hooks.example.com/services/{MARK}", + "headers": {"X-Api-Key": MARK, "X-Signature": {"from_env": "HOOK_SIGNATURE"}}, + } + ) + ], + ) + stub = runs_pb2_grpc.RunServiceStub(channel) + submitted = await stub.SubmitRun(runs_pb2.SubmitRunRequest(spec=spec)) + run_id = submitted.run.run_id + responses = [ + submitted, + await stub.GetRun(runs_pb2.GetRunRequest(run_id=run_id)), + await stub.ListRuns(runs_pb2.ListRunsRequest()), + await stub.CancelRun(runs_pb2.CancelRunRequest(run_id=run_id)), + ] + for response in responses: + assert MARK.encode() not in response.SerializeToString(), type(response).__name__ + + run = responses[1].run + assert run.spec.target.http.base_url == "https://traces.example.com/api" + assert run.spec.context.headers["Authorization"] == "REDACTED" + sink = convert.from_struct(run.spec.sinks[0]) + assert sink["url"] == "https://hooks.example.com/REDACTED" + assert sink["headers"] == {"X-Api-Key": "REDACTED", "X-Signature": {"from_env": "HOOK_SIGNATURE"}} + + stored = await repos.runs.get(UUID(run_id)) + assert stored.spec.context["headers"]["Authorization"] == f"Bearer {MARK}" + + async def test_conflict_names_the_run_without_its_spec(self, channel): + stub = runs_pb2_grpc.RunServiceStub(channel) + run_id = str(uuid4()) + first = _run_spec(context=runs_pb2.RunContext(headers={"Authorization": f"Bearer {MARK}"})) + await stub.SubmitRun(runs_pb2.SubmitRunRequest(run_id=run_id, spec=first)) + assert (await stub.SubmitRun(runs_pb2.SubmitRunRequest(run_id=run_id, spec=first))).run.run_id == run_id + + with pytest.raises(grpc.aio.AioRpcError) as error: + await stub.SubmitRun(runs_pb2.SubmitRunRequest(run_id=run_id, spec=_run_spec())) + assert error.value.code() == grpc.StatusCode.ALREADY_EXISTS + info = _error_info(error.value) + assert (info.reason, dict(info.metadata)) == ("RUN_SPEC_CONFLICT", {"run_id": run_id}) + assert MARK.encode() not in _trailers(error.value) + + @pytest.mark.parametrize("trace_id", ["../../admin", "abc", TRACE_ID + "0", f"{TRACE_ID[:16]}/x"]) + async def test_http_target_needs_a_hex_trace_id(self, channel, trace_id): + spec = runs_pb2.RunSpec( + target=runs_pb2.TraceTarget( + http=runs_pb2.HttpTraceTarget(base_url="https://traces.example.com/api", trace_id=trace_id) + ), + eval_config=_trajectory_params(), + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await runs_pb2_grpc.RunServiceStub(channel).SubmitRun(runs_pb2.SubmitRunRequest(spec=spec)) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert "trace_id" in error.value.details() + + async def test_uploaded_target_is_output_only(self, channel): + spec = runs_pb2.RunSpec(target=runs_pb2.TraceTarget(uploaded=runs_pb2.UploadedTraces(trace_count=1))) + with pytest.raises(grpc.aio.AioRpcError) as error: + await runs_pb2_grpc.RunServiceStub(channel).SubmitRun(runs_pb2.SubmitRunRequest(spec=spec)) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + + @pytest.mark.parametrize("run_id", ["", "not-a-uuid"]) + async def test_get_run_needs_a_uuid(self, channel, run_id): + with pytest.raises(grpc.aio.AioRpcError) as error: + await runs_pb2_grpc.RunServiceStub(channel).GetRun(runs_pb2.GetRunRequest(run_id=run_id)) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + + async def test_get_unknown_run_is_not_found(self, channel): + with pytest.raises(grpc.aio.AioRpcError) as error: + await runs_pb2_grpc.RunServiceStub(channel).GetRun(runs_pb2.GetRunRequest(run_id=str(uuid4()))) + assert error.value.code() == grpc.StatusCode.NOT_FOUND + + async def test_list_runs_pages_newest_first_and_filters_by_status(self, channel, repos): + start = datetime(2026, 1, 1, tzinfo=UTC) + ids = [] + for n, status in enumerate([RunStatus.QUEUED, RunStatus.SUCCEEDED, RunStatus.QUEUED]): + run = Run( + run_id=uuid4(), + status=status, + spec=RunSpec(target=TraceTarget(kind="inline", inline={"n": n})), + created_at=start + timedelta(minutes=n), + ) + await repos.runs.create(run) + ids.append(str(run.run_id)) + stub = runs_pb2_grpc.RunServiceStub(channel) + + page = await stub.ListRuns(runs_pb2.ListRunsRequest(page=common_pb2.PageRequest(limit=2))) + assert [r.run_id for r in page.runs] == [ids[2], ids[1]] + rest = await stub.ListRuns( + runs_pb2.ListRunsRequest(page=common_pb2.PageRequest(limit=2, page_token=page.page.next_page_token)) + ) + assert ([r.run_id for r in rest.runs], rest.page.next_page_token) == ([ids[0]], "") + + done = await stub.ListRuns(runs_pb2.ListRunsRequest(statuses=[runs_pb2.RUN_STATUS_SUCCEEDED])) + assert [r.run_id for r in done.runs] == [ids[1]] + + async def test_cancel_a_queued_run(self, channel): + stub = runs_pb2_grpc.RunServiceStub(channel) + run = (await stub.SubmitRun(runs_pb2.SubmitRunRequest(spec=_run_spec()))).run + cancelled = (await stub.CancelRun(runs_pb2.CancelRunRequest(run_id=run.run_id))).run + assert (cancelled.status, cancelled.cancel_requested) == (runs_pb2.RUN_STATUS_CANCELLED, True) + assert cancelled.HasField("finished_at") + + async def test_results_keep_unscored_invocations_in_place(self, channel, repos): + stub = runs_pb2_grpc.RunServiceStub(channel) + run_id = UUID((await stub.SubmitRun(runs_pb2.SubmitRunRequest(spec=_run_spec()))).run.run_id) + await repos.results.upsert_many(run_id, [_result(run_id, n, [0.5, None, 1.0]) for n in range(3)]) + + page = await stub.ListRunResults( + runs_pb2.ListRunResultsRequest(run_id=str(run_id), page=common_pb2.PageRequest(limit=2)) + ) + assert [r.result_id for r in page.results] == ["r0", "r1"] + assert [(i.HasField("score"), i.score) for i in page.results[0].per_invocation] == [ + (True, 0.5), + (False, 0.0), + (True, 1.0), + ] + rest = await stub.ListRunResults( + runs_pb2.ListRunResultsRequest( + run_id=str(run_id), page=common_pb2.PageRequest(limit=2, page_token=page.page.next_page_token) + ) + ) + assert ([r.result_id for r in rest.results], rest.page.next_page_token) == (["r2"], "") + + async def test_watch_run_streams_each_change_until_the_run_ends(self, channel, repos, monkeypatch): + monkeypatch.setattr(services, "WATCH_RUN_POLL_SECONDS", 0.01) + stub = runs_pb2_grpc.RunServiceStub(channel) + run_id = UUID((await stub.SubmitRun(runs_pb2.SubmitRunRequest(spec=_run_spec()))).run.run_id) + call = stub.WatchRun(runs_pb2.WatchRunRequest(run_id=str(run_id))) + + first = await asyncio.wait_for(call.read(), 5) + assert (first.run.status, len(first.results)) == (runs_pb2.RUN_STATUS_QUEUED, 0) + await repos.runs.claim_next(worker_id="w", lease=timedelta(seconds=30), max_attempts=3) + await repos.results.upsert_many(run_id, [_result(run_id, 0, [1.0])]) + await repos.runs.update_status(run_id, RunStatus.SUCCEEDED, summary={"passed": 1}) + + updates = [first, *await asyncio.wait_for(_read_to_end(call), 5)] + statuses = [u.run.status for u in updates] + assert statuses[-1] == runs_pb2.RUN_STATUS_SUCCEEDED + assert statuses == sorted(statuses) + assert [r.result_id for u in updates for r in u.results] == ["r0"] + assert convert.from_struct(updates[-1].run.summary) == {"passed": 1} + + async def test_watch_unknown_run_is_not_found(self, channel): + call = runs_pb2_grpc.RunServiceStub(channel).WatchRun(runs_pb2.WatchRunRequest(run_id=str(uuid4()))) + with pytest.raises(grpc.aio.AioRpcError) as error: + await _stream(call) + assert error.value.code() == grpc.StatusCode.NOT_FOUND + + +def _result(run_id: UUID, n: int, scores: list[float | None]) -> Result: + return Result( + result_id=f"r{n}", + run_id=run_id, + eval_set_item_id=f"item-{n}", + eval_set_item_name=f"trace-{n}", + evaluator_name="tool_trajectory_avg_score", + evaluator_type="builtin", + status=ResultStatus.PASSED, + score=1.0, + per_invocation_scores=scores, + ) + + +class TestConversion: + async def test_convert_returns_turns_and_a_golden_eval_set(self, channel): + response = await conversion_pb2_grpc.ConversionServiceStub(channel).ConvertTraces( + conversion_pb2.ConvertTracesRequest(traces=common_pb2.TraceInput(documents=[_document()])) + ) + [converted] = response.conversations + assert [t.user_text for t in converted.conversation.turns] == ["What is the weather in Paris?"] + golden = convert.from_struct(converted.golden_adk_eval_set) + response_data = golden["eval_cases"][0]["conversation"][0]["intermediate_data"]["tool_responses"][0] + assert response_data["response"]["temp_c"] == 21 + assert isinstance(response_data["response"]["temp_c"], int) + assert list(response.warnings) == [] + + async def test_otlp_protobuf_input_equals_the_json_document(self, channel): + stub = conversion_pb2_grpc.ConversionServiceStub(channel) + from_json = await stub.ConvertTraces( + conversion_pb2.ConvertTracesRequest(traces=common_pb2.TraceInput(documents=[_document()])) + ) + from_proto = await stub.ConvertTraces( + conversion_pb2.ConvertTracesRequest( + traces=common_pb2.TraceInput(otlp=common_pb2.OtlpTelemetry(traces=_traces_data(DOCUMENT))) + ) + ) + assert from_proto.conversations == from_json.conversations + assert len(from_json.conversations[0].conversation.turns) == 1 + + async def test_an_unreadable_document_is_a_warning(self, channel): + response = await conversion_pb2_grpc.ConversionServiceStub(channel).ConvertTraces( + conversion_pb2.ConvertTracesRequest( + traces=common_pb2.TraceInput(documents=[_document(b"\xff not json", name="broken.json"), _document()]) + ) + ) + assert len(response.conversations) == 1 + [warning] = response.warnings + assert warning.startswith("Failed to load broken.json") + + async def test_an_oversized_document_is_invalid_argument(self, channel): + big = _document(b" " * (convert.MAX_DOCUMENT_BYTES + 1), name="big.json") + with pytest.raises(grpc.aio.AioRpcError) as error: + await conversion_pb2_grpc.ConversionServiceStub(channel).ConvertTraces( + conversion_pb2.ConvertTracesRequest(traces=common_pb2.TraceInput(documents=[big])) + ) + assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert "big.json" in error.value.details() + + +class TestSystem: + async def test_get_version(self, channel): + response = await system_pb2_grpc.SystemServiceStub(channel).GetVersion(system_pb2.GetVersionRequest()) + assert response.version == __version__ + + async def test_get_config(self, channel, context, monkeypatch): + for name in ("GOOGLE_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENAI_API_KEY", MARK) + response = await system_pb2_grpc.SystemServiceStub(channel).GetConfig(system_pb2.GetConfigRequest()) + assert (response.ports.http, response.ports.otlp_http, response.ports.otlp_grpc, response.ports.api_grpc) == ( + 8001, + 4318, + 4317, + 8002, + ) + limits = context.manager.store.limits + assert response.live_limits.max_sessions == limits.max_sessions + assert response.live_limits.idle_timeout.ToTimedelta() == timedelta(seconds=limits.idle_timeout_seconds) + assert response.runs_enabled is True + assert (response.judge_keys.openai, response.judge_keys.google, response.judge_keys.anthropic) == ( + True, + False, + False, + ) + assert MARK.encode() not in response.SerializeToString() diff --git a/tests/api/test_redact.py b/tests/api/test_redact.py new file mode 100644 index 00000000..af325cbd --- /dev/null +++ b/tests/api/test_redact.py @@ -0,0 +1,78 @@ +"""Secrets out of run specs on every read path.""" + +from __future__ import annotations + +import pytest + +from agentevals.api.redact import REDACTED, redact_spec +from agentevals.storage.models import RunSpec, TraceTarget + + +def _spec(**fields) -> RunSpec: + return RunSpec(target=fields.pop("target", TraceTarget(kind="inline", inline={})), **fields) + + +class TestHeaders: + def test_literal_values_are_redacted_and_env_references_kept(self): + spec = _spec( + context={"headers": {"Authorization": "Bearer a", "X-Ref": {"from_env": "TOKEN_VAR"}}}, + sinks=[{"kind": "http_webhook", "url": "https://h.example.com", "headers": {"X-Key": "b"}}], + ) + out = redact_spec(spec) + assert out.context["headers"] == {"Authorization": REDACTED, "X-Ref": {"from_env": "TOKEN_VAR"}} + assert out.sinks[0]["headers"] == {"X-Key": REDACTED} + + def test_a_reference_with_extra_keys_is_treated_as_a_value(self): + out = redact_spec(_spec(context={"headers": {"X": {"from_env": "V", "value": "literal"}}})) + assert out.context["headers"]["X"] == REDACTED + + +class TestSecretKeys: + @pytest.mark.parametrize( + "key", ["token", "api_key", "apiKey", "client_secret", "password", "dsn", "private_key", "Authorization"] + ) + def test_secret_like_keys_are_redacted_at_any_depth(self, key): + out = redact_spec(_spec(sinks=[{"kind": "custom", "auth": {key: "value", "region": "eu"}}])) + assert out.sinks[0]["auth"] == {key: REDACTED, "region": "eu"} + + def test_other_keys_and_nulls_are_kept(self): + out = redact_spec(_spec(sinks=[{"kind": "file", "path": "/var/out.ndjson", "token": None}])) + assert out.sinks[0] == {"kind": "file", "path": "/var/out.ndjson", "token": None} + + +class TestUrls: + @pytest.mark.parametrize( + ("url", "expected"), + [ + ("https://hooks.example.com/services/T0/B0/abc", "https://hooks.example.com/REDACTED"), + ("https://hooks.example.com?sig=abc", "https://hooks.example.com/REDACTED"), + ("https://hooks.example.com:8443/", "https://hooks.example.com:8443"), + ("https://user:pw@hooks.example.com/x", "https://hooks.example.com/REDACTED"), + ("not a url", REDACTED), + ], + ) + def test_sink_urls_keep_only_their_origin(self, url, expected): + assert redact_spec(_spec(sinks=[{"kind": "http_webhook", "url": url}])).sinks[0]["url"] == expected + + @pytest.mark.parametrize( + ("base_url", "expected"), + [ + ("https://reader:pw@traces.example.com:9443/api?key=k", "https://traces.example.com:9443/api"), + ("https://traces.example.com/api?key=k", "https://traces.example.com/api"), + ("https://traces.example.com/api", "https://traces.example.com/api"), + ], + ) + def test_trace_base_url_loses_userinfo_and_query(self, base_url, expected): + target = TraceTarget(kind="http", base_url=base_url, trace_id="1dbd779ff94bf24c") + assert redact_spec(_spec(target=target)).target.base_url == expected + + +def test_the_spec_itself_is_not_changed(): + spec = _spec( + target=TraceTarget(kind="http", base_url="https://u:p@t.example.com/api", trace_id="1dbd779ff94bf24c"), + context={"headers": {"Authorization": "Bearer a"}}, + sinks=[{"kind": "http_webhook", "url": "https://h.example.com/x", "headers": {"X-Key": "b"}}], + ) + before = spec.model_dump() + redact_spec(spec) + assert spec.model_dump() == before diff --git a/tests/api/test_rpc_convert.py b/tests/api/test_rpc_convert.py new file mode 100644 index 00000000..ae42bc7d --- /dev/null +++ b/tests/api/test_rpc_convert.py @@ -0,0 +1,202 @@ +"""Conversions between the v1alpha1 messages and the Pydantic models.""" + +from __future__ import annotations + +import math +import typing +from datetime import UTC, datetime +from uuid import uuid4 + +import grpc +import pytest +from google.adk.evaluation.eval_metrics import ToolTrajectoryCriterion +from google.adk.evaluation.evaluator import EvalStatus + +from agentevals.api.rpc import convert +from agentevals.api.rpc.policy import ApiError +from agentevals.api.v1alpha1 import common_pb2, evaluation_pb2, runs_pb2 +from agentevals.config import BuiltinMetricDef, CodeEvaluatorDef, EvalParams, GroupBy, RemoteEvaluatorDef +from agentevals.runner import MetricResult +from agentevals.storage.models import Result, ResultStatus, Run, RunSpec, RunStatus, TraceTarget + + +def _proto_values(enum) -> set[int]: + return {v.number for v in enum.DESCRIPTOR.values if not v.name.endswith("_UNSPECIFIED")} + + +class TestEnumMaps: + def test_run_status(self): + assert set(convert.RUN_STATUS) == set(RunStatus) + assert set(convert.RUN_STATUS.values()) == _proto_values(runs_pb2.RunStatus) + + def test_result_status(self): + assert set(convert.RESULT_STATUS) == set(ResultStatus) + assert set(convert.RESULT_STATUS.values()) == _proto_values(runs_pb2.ResultStatus) + + def test_evaluator_type(self): + assert set(convert.EVALUATOR_TYPE) == set(typing.get_args(Result.model_fields["evaluator_type"].annotation)) + assert set(convert.EVALUATOR_TYPE.values()) == _proto_values(runs_pb2.EvaluatorType) + + def test_eval_status(self): + assert set(convert.EVAL_STATUS) == {s.name for s in EvalStatus} + assert set(convert.EVAL_STATUS.values()) == _proto_values(evaluation_pb2.EvalStatus) + + def test_group_by(self): + assert set(convert.GROUP_BY) == set(typing.get_args(GroupBy)) + assert set(convert.GROUP_BY.values()) == _proto_values(common_pb2.GroupBy) + + def test_trajectory_match_type(self): + assert set(convert.TRAJECTORY_MATCH) == {m.name for m in ToolTrajectoryCriterion.MatchType} + assert set(convert.TRAJECTORY_MATCH.values()) == _proto_values(evaluation_pb2.TrajectoryMatchType) + + def test_trace_format(self): + [literal, _none] = typing.get_args(TraceTarget.model_fields["trace_format"].annotation) + assert set(convert.TRACE_FORMAT) == set(typing.get_args(literal)) + assert set(convert.TRACE_FORMAT_FROM_PB) == {v.number for v in common_pb2.TraceFormat.DESCRIPTOR.values} + + +class TestStruct: + def test_whole_numbers_come_back_as_ints(self): + data = {"count": 3, "ratio": 0.25, "nested": [1, {"n": -2}], "flag": True, "none": None} + back = convert.from_struct(convert.to_struct(data)) + assert back == data + assert type(back["count"]) is int and type(back["nested"][1]["n"]) is int + assert type(back["ratio"]) is float and type(back["flag"]) is bool + + def test_values_a_double_cannot_hold_exactly_become_text(self): + back = convert.from_struct(convert.to_struct({"big": 2**60, "nan": math.nan, "raw": b"\x00\x01"})) + assert back == {"big": str(2**60), "nan": None, "raw": "AAE="} + + +class TestEvalParams: + def test_round_trip(self): + params = EvalParams( + evaluators=[ + BuiltinMetricDef(name="tool_trajectory_avg_score", threshold=0.8, trajectory_match_type="IN_ORDER"), + BuiltinMetricDef( + name="final_response_match_v2", + judge_model="gemini-2.5-flash", + credential_ref="judge", + judge_base_url="https://llm.example.com/v1", + ), + CodeEvaluatorDef(name="length", path="evals/length.py", threshold=0.4, timeout=12, config={"max": 3}), + RemoteEvaluatorDef(name="tone", source="github", ref="evaluators/tone/tone.py"), + ], + max_concurrent_traces=2, + max_concurrent_evals=3, + group_by="conversation", + ) + back = convert.eval_params_from_pb(convert.eval_params_to_pb(params)) + assert back == params + assert type(back.evaluators[2].config["max"]) is int + + def test_unset_fields_keep_their_defaults(self): + assert convert.eval_params_from_pb(evaluation_pb2.EvalParams()) == EvalParams() + + def test_stored_parameters_with_unknown_keys_still_convert(self): + stored = {**EvalParams().model_dump(), "trace_files": ["/tmp/x.json"]} + assert convert.eval_params_from_pb(convert.eval_params_to_pb(stored)) == EvalParams() + + def test_evaluator_needs_a_kind(self): + with pytest.raises(ApiError) as error: + convert.eval_params_from_pb(evaluation_pb2.EvalParams(evaluators=[evaluation_pb2.Evaluator()])) + assert error.value.code == grpc.StatusCode.INVALID_ARGUMENT + + +def _spec(target: TraceTarget) -> RunSpec: + return RunSpec( + target=target, + eval_set={"eval_set_id": "golden", "eval_cases": [{"eval_id": "c1", "conversation": []}]}, + eval_config=EvalParams(max_concurrent_evals=2).model_dump(by_alias=False), + sinks=[{"kind": "stdout"}, {"kind": "file", "path": "/var/out.ndjson"}], + context={"headers": {"X-Tenant": "team-a"}}, + credential_refs={"judge": {"kind": "kubernetes", "name": "judge-key", "key": "api_key"}}, + ) + + +class TestRunSpec: + @pytest.mark.parametrize( + "target", + [ + TraceTarget(kind="inline", inline={"resourceSpans": []}, trace_format="otlp-json"), + TraceTarget( + kind="http", + base_url="https://traces.example.com/api", + trace_id="1dbd779ff94bf24c7b84678a5dc0346c", + trace_format="jaeger-json", + ), + ], + ) + def test_round_trip(self, target): + spec = _spec(target) + assert convert.run_spec_from_pb(convert.run_spec_to_pb(spec)) == spec + + def test_uploaded_target_is_output_only(self): + spec = convert.run_spec_to_pb(_spec(TraceTarget(kind="uploaded", trace_count=1, trace_files=["a.json"]))) + assert spec.target.uploaded.trace_files == ["a.json"] + with pytest.raises(ApiError, match="output only"): + convert.run_spec_from_pb(spec) + + @pytest.mark.parametrize(("content", "message"), [(b"{", "one JSON document"), (b"[]", "JSON object")]) + def test_inline_target_must_be_a_json_object(self, content, message): + spec = runs_pb2.RunSpec(target=runs_pb2.TraceTarget(inline=common_pb2.TraceDocument(content=content))) + with pytest.raises(ApiError, match=message): + convert.run_spec_from_pb(spec) + + def test_target_is_required(self): + with pytest.raises(ApiError, match="target needs one of"): + convert.run_spec_from_pb(runs_pb2.RunSpec()) + + def test_run_to_pb_redacts(self): + spec = _spec(TraceTarget(kind="inline", inline={})) + spec.context["headers"]["Authorization"] = "Bearer qxmark" + run = Run(run_id=uuid4(), status=RunStatus.QUEUED, spec=spec) + out = convert.run_to_pb(run) + assert out.spec.context.headers["Authorization"] == "REDACTED" + assert b"qxmark" not in out.SerializeToString() + assert run.spec.context["headers"]["Authorization"] == "Bearer qxmark" + + def test_naive_times_are_utc(self): + run = Run(run_id=uuid4(), status=RunStatus.QUEUED, spec=_spec(TraceTarget(kind="inline", inline={}))) + run.created_at = datetime(2026, 3, 1, 12, 30) + assert convert.run_to_pb(run).created_at.ToDatetime(tzinfo=UTC) == datetime(2026, 3, 1, 12, 30, tzinfo=UTC) + + +class TestResults: + def test_unscored_invocations_keep_their_position(self): + result = Result( + result_id="r1", + run_id=uuid4(), + eval_set_item_id="i", + eval_set_item_name="n", + evaluator_name="m", + evaluator_type="code", + status=ResultStatus.ERRORED, + score=math.nan, + per_invocation_scores=[None, 0.5], + latency_ms=0, + ) + out = convert.result_to_pb(result) + assert [i.HasField("score") for i in out.per_invocation] == [False, True] + assert out.per_invocation[1].score == 0.5 + assert not out.HasField("score") + assert out.HasField("latency_ms") + assert (out.evaluator_type, out.status) == (runs_pb2.EVALUATOR_TYPE_CODE, runs_pb2.RESULT_STATUS_ERRORED) + + def test_metric_result_pairs_scores_with_statuses(self): + metric = MetricResult( + metric_name="m", + score=0.0, + eval_status="FAILED", + per_invocation_scores=[1.0, None], + per_invocation_statuses=["PASSED", "NOT_EVALUATED", "FAILED"], + duration_ms=math.inf, + ) + out = convert.metric_result_to_pb(metric) + assert out.HasField("score") and out.score == 0.0 + assert [(i.HasField("score"), i.status) for i in out.per_invocation] == [ + (True, evaluation_pb2.EVAL_STATUS_PASSED), + (False, evaluation_pb2.EVAL_STATUS_NOT_EVALUATED), + (False, evaluation_pb2.EVAL_STATUS_FAILED), + ] + assert not out.HasField("duration_ms") diff --git a/tests/api/test_runs_routes.py b/tests/api/test_runs_routes.py index d0a35e78..27decf3f 100644 --- a/tests/api/test_runs_routes.py +++ b/tests/api/test_runs_routes.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest from fastapi.testclient import TestClient @@ -192,3 +192,78 @@ def test_get_run_results_for_unknown_run_404(self, stubbed_app): with TestClient(app) as client: r = client.get(f"/api/runs/{uuid4()}/results") assert r.status_code == 404 + + +MARK = "qxmark" + + +def _secret_payload(**spec): + return { + "spec": { + "approach": "trace_replay", + "target": { + "kind": "http", + "baseUrl": f"https://reader:{MARK}@traces.example.com/api?key={MARK}", + "traceId": "1dbd779ff94bf24c7b84678a5dc0346c", + }, + "context": {"headers": {"Authorization": f"Bearer {MARK}"}}, + "sinks": [ + { + "kind": "http_webhook", + "url": f"https://hooks.example.com/services/{MARK}", + "headers": {"X-Api-Key": MARK, "X-Signature": {"from_env": "HOOK_SIGNATURE"}}, + } + ], + **spec, + } + } + + +class TestSecretsStayOutOfResponses: + def test_every_read_is_redacted(self, stubbed_app): + app, repos = stubbed_app + with TestClient(app) as client: + submitted = client.post("/api/runs", json=_secret_payload()) + run_id = submitted.json()["data"]["runId"] + responses = [ + submitted, + client.get(f"/api/runs/{run_id}"), + client.get("/api/runs"), + client.post(f"/api/runs/{run_id}/cancel"), + ] + for response in responses: + assert response.status_code in (200, 202) + assert MARK not in response.text + spec = responses[1].json()["data"]["spec"] + assert spec["target"]["baseUrl"] == "https://traces.example.com/api" + assert spec["context"]["headers"]["Authorization"] == "REDACTED" + assert spec["sinks"][0]["url"] == "https://hooks.example.com/REDACTED" + assert spec["sinks"][0]["headers"]["X-Signature"] == {"from_env": "HOOK_SIGNATURE"} + + async def test_the_stored_run_keeps_the_real_values(self, stubbed_app): + app, repos = stubbed_app + with TestClient(app) as client: + run_id = client.post("/api/runs", json=_secret_payload()).json()["data"]["runId"] + stored = await repos.runs.get(UUID(run_id)) + assert stored.spec.context["headers"]["Authorization"] == f"Bearer {MARK}" + assert stored.spec.sinks[0]["url"] == f"https://hooks.example.com/services/{MARK}" + + def test_conflict_body_is_redacted(self, stubbed_app): + app, _ = stubbed_app + run_id = "44444444-4444-4444-4444-444444444444" + with TestClient(app) as client: + client.post("/api/runs", json={**_secret_payload(), "runId": run_id}) + r = client.post("/api/runs", json={**_secret_payload(eval_set={"eval_set_id": "x"}), "runId": run_id}) + assert r.status_code == 409 + assert MARK not in r.text + assert r.json()["detail"]["persisted"]["runId"] == run_id + + @pytest.mark.parametrize("trace_id", ["../../admin", "1dbd779ff94bf24c/x", "1dbd779ff94bf24"]) + def test_http_target_needs_a_hex_trace_id(self, stubbed_app, trace_id): + app, _ = stubbed_app + payload = _secret_payload() + payload["spec"]["target"]["traceId"] = trace_id + with TestClient(app) as client: + r = client.post("/api/runs", json=payload) + assert r.status_code == 422 + assert "trace_id" in r.json()["detail"] diff --git a/tests/api/test_views.py b/tests/api/test_views.py new file mode 100644 index 00000000..bad4675d --- /dev/null +++ b/tests/api/test_views.py @@ -0,0 +1,232 @@ +"""Session and turn views: what every client renders without deriving it.""" + +from __future__ import annotations + +import copy +import json +import math +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from google.protobuf import json_format +from opentelemetry.proto.trace.v1 import trace_pb2 + +from agentevals.api.v1alpha1 import views_pb2 +from agentevals.api.views import ( + GAP_NO_TURNS, + GAP_TRUNCATED, + MAX_TEXT_CHARS, + MAX_VALUE_BYTES, + MAX_VALUE_DEPTH, + SessionFacts, + session_view, + turn_view, +) +from agentevals.genai.extract import extract_conversation +from agentevals.genai.model import Gap, ToolCall, Turn, Usage +from agentevals.loader import load_telemetry, load_telemetry_text +from agentevals.otel.model import SpanRef + +CORPUS = Path(__file__).resolve().parents[1] / "fixtures" / "otlp" +FIXTURE = CORPUS / "hand" / "adk_go_main.json" +TRUTHS = sorted(CORPUS.rglob("*.truth.json")) +DOCUMENT = json.loads(FIXTURE.read_text()) +TRACE_ID = "1dbd779ff94bf24c7b84678a5dc0346c" +RECEIVED = datetime(2026, 10, 1, 9, 0, tzinfo=UTC) + + +def _facts(session_id: str = "s1", *, complete: bool = True, span_count: int = 5) -> SessionFacts: + return SessionFacts( + session_id=session_id, + received_at=RECEIVED, + is_complete=complete, + trace_ids=(TRACE_ID,), + span_count=span_count, + ) + + +def _session(document: dict = DOCUMENT, facts: SessionFacts | None = None) -> views_pb2.Session: + traces = load_telemetry_text(json.dumps(document), format="otlp-json").traces + conversation = extract_conversation(traces, "s1") + spans = {(s.trace_id, s.span_id): s for t in traces for s in t.spans.values()} + return session_view(facts or _facts(), conversation, spans) + + +def _with_resource(**attributes: str) -> dict: + document = copy.deepcopy(DOCUMENT) + for resource_spans in document["resourceSpans"]: + resource_spans["resource"]["attributes"] += [ + {"key": k, "value": {"stringValue": v}} for k, v in attributes.items() + ] + return document + + +def _turn(**fields) -> Turn: + return Turn(ref=SpanRef(TRACE_ID, "eb86921785ed9606"), **fields) + + +def _user(text: str) -> list[dict]: + return [{"role": "user", "parts": [{"type": "text", "content": text}]}] + + +class TestShortId: + @pytest.mark.parametrize( + ("session_id", "short_id"), + [ + ("0199a3c2-6f1e-7c4b-9d2a-5e8f1a2b3c4d", "2b3c4d"), + ("0199a3c26f1e7c4b9d2a5e8f1a2b3c4d", "2b3c4d"), + (TRACE_ID, "c0346c"), + ("o-code-review", "o-code-review"), + ("o-code", "o-code"), + ("abc123", "abc123"), + ("deadbeefcafe", "deadbeefcafe"), + ], + ) + def test_only_generated_ids_are_shortened(self, session_id, short_id): + assert session_view(_facts(session_id)).summary.short_id == short_id + + +class TestSummary: + def test_totals_come_from_the_turns(self): + summary = _session().summary + totals = summary.totals + assert (totals.turns, totals.failed_turns, totals.tool_calls) == (1, 0, 1) + assert totals.model_calls == totals.model_calls_with_usage == 2 + assert totals.tool_calls_with_arguments == 1 + assert list(summary.models) == ["gemini-2.5-flash"] + assert summary.first_user_text == "What is the weather in Paris?" + assert summary.content_capture == views_pb2.CONTENT_CAPTURE_ON + assert summary.state == views_pb2.SESSION_STATE_COMPLETE + assert summary.conversation_id == "sess-go-1" + + def test_unrecorded_token_counts_stay_unset(self): + session = _session() + for usage in (session.summary.totals.usage, session.turns[0].usage): + assert (usage.input_tokens, usage.output_tokens) == (90, 18) + for name in ("cache_read_input_tokens", "cache_write_input_tokens", "reasoning_output_tokens"): + assert not usage.HasField(name), name + assert "cacheReadInputTokens" not in json_format.MessageToDict(session.summary.totals.usage) + + @pytest.mark.parametrize( + ("resource", "agent"), + [({}, "weather_agent"), ({"gen_ai.agent.name": "planner"}, "planner")], + ) + def test_agent_prefers_the_resource_then_the_turn(self, resource, agent): + assert _session(_with_resource(**resource)).summary.agent == agent + + def test_agent_falls_back_to_the_service(self): + document = copy.deepcopy(DOCUMENT) + for resource_spans in document["resourceSpans"]: + for scope_spans in resource_spans["scopeSpans"]: + for span in scope_spans["spans"]: + span["attributes"] = [a for a in span["attributes"] if a["key"] != "gen_ai.agent.name"] + assert _session(document).summary.agent == "fixture-agent" + + def test_agent_version_falls_back_to_the_service_version(self): + assert _session().summary.agent_version == "1.0.0" + assert _session(_with_resource(**{"gen_ai.agent.version": "7"})).summary.agent_version == "7" + + def test_resource_is_carried_as_otlp(self): + attributes = {a.key: a.value.string_value for a in _session().summary.resource.attributes} + assert attributes == {"service.name": "fixture-agent", "service.version": "1.0.0"} + + def test_a_session_without_turns_is_described_by_its_facts(self): + summary = session_view(_facts(complete=False)).summary + assert summary.state == views_pb2.SESSION_STATE_LIVE + assert summary.started_at.ToDatetime(tzinfo=UTC) == RECEIVED + assert (summary.span_count, list(summary.trace_ids)) == (5, [TRACE_ID]) + assert not summary.HasField("first_user_text") + assert list(summary.gaps) == [] + + def test_a_complete_session_without_turns_says_so(self): + [gap] = session_view(_facts()).summary.gaps + assert gap.code == GAP_NO_TURNS + + def test_summary_only_leaves_the_turns_out(self): + traces = load_telemetry_text(FIXTURE.read_text(), format="otlp-json").traces + session = session_view(_facts(), extract_conversation(traces), {}, include_turns=False) + assert (len(session.turns), session.summary.totals.turns) == (0, 1) + + +class TestTurn: + def test_texts_and_status(self): + view = turn_view(_turn(user_input=_user("hi"), status="error", error_type="TimeoutError"), {}) + assert (view.user_text, view.status, view.error_type) == ( + "hi", + trace_pb2.Status.STATUS_CODE_ERROR, + "TimeoutError", + ) + assert not view.HasField("answer_text") + + def test_long_text_is_cut_with_a_gap(self): + view = turn_view(_turn(user_input=_user("x" * (MAX_TEXT_CHARS + 10))), {}) + assert len(view.user_text) == MAX_TEXT_CHARS + [gap] = view.gaps + assert gap.code == GAP_TRUNCATED and "user text" in gap.reason + + def test_large_tool_result_becomes_cut_json_text(self): + result = {"rows": ["y" * 1000 for _ in range(100)]} + view = turn_view(_turn(tool_calls=[ToolCall(name="query", arguments={"q": 1}, result=result)]), {}) + [call] = view.tool_calls + assert call.result.WhichOneof("kind") == "string_value" + assert len(call.result.string_value.encode()) <= MAX_VALUE_BYTES + assert json.dumps(result, separators=(",", ":")).startswith(call.result.string_value) + assert json_format.MessageToDict(call.arguments) == {"q": 1} + [gap] = view.gaps + assert gap.code == GAP_TRUNCATED and "tool query result" in gap.reason + + def test_values_json_cannot_carry_are_replaced_with_a_gap(self): + deep: dict = {} + node = deep + for _ in range(MAX_VALUE_DEPTH + 5): + node["n"] = {} + node = node["n"] + arguments = {"nan": math.nan, "raw": b"\x01\x02", "big": 2**64, "deep": deep} + view = turn_view(_turn(tool_calls=[ToolCall(name="t", arguments=arguments)]), {}) + args = json_format.MessageToDict(view.tool_calls[0].arguments) + assert (args["nan"], args["raw"], args["big"]) == (None, "AQI=", str(2**64)) + assert json_format.MessageToJson(view) + [gap] = view.gaps + for reason in ("non finite", "base64", "2^53", "nesting"): + assert reason in gap.reason + + def test_unknown_results_stay_unset(self): + [call] = turn_view(_turn(tool_calls=[ToolCall(name="t")]), {}).tool_calls + assert not call.HasField("arguments") and not call.HasField("result") and not call.HasField("call_id") + + def test_repeated_warnings_become_one_gap(self): + gap = Gap(code="tool_result_by_order", reason="paired by order") + view = turn_view(_turn(warnings=[gap, gap]), {}) + assert [(g.code, g.reason) for g in view.gaps] == [("tool_result_by_order", "paired by order")] + + +class TestUsage: + def test_sum_keeps_unrecorded_counts_unset(self): + total = Usage(input_tokens=5) + Usage(output_tokens=2) + Usage() + assert total == Usage(input_tokens=5, output_tokens=2) + + def test_recorded_zero_is_kept(self): + assert (Usage(input_tokens=0) + Usage()).input_tokens == 0 + + +@pytest.mark.parametrize("truth_path", TRUTHS, ids=lambda p: str(p.relative_to(CORPUS)).removesuffix(".truth.json")) +def test_every_corpus_fixture_has_a_view_that_matches_its_truth(truth_path): + truth = json.loads(truth_path.read_text()) + fixture = truth_path.with_name(truth_path.name.replace(".truth.json", ".json")) + traces = load_telemetry(str(fixture)).traces + conversation = extract_conversation(traces, "s1") + spans = {(s.trace_id, s.span_id): s for t in traces for s in t.spans.values()} + session = session_view(_facts(), conversation, spans) + + assert json_format.Parse(json_format.MessageToJson(session), views_pb2.Session()) == session + totals = session.summary.totals + if truth.get("extra_turns_allowed"): + assert totals.turns >= len(truth["turns"]) + else: + assert totals.turns == len(truth["turns"]) == len(session.turns) + for name, key in (("input_tokens", "input"), ("output_tokens", "output")): + recorded = any(getattr(t.usage, name) is not None for t in conversation.turns) + assert totals.usage.HasField(name) is recorded + assert getattr(totals.usage, name) == truth["tokens"][key] + assert totals.tool_calls == sum(len(t.tool_calls) for t in session.turns) diff --git a/tests/fixtures/otlp/hand/adk_cycle_poisons_batch.truth.json b/tests/fixtures/otlp/hand/adk_cycle_poisons_batch.truth.json index b63982c1..43c32c7e 100644 --- a/tests/fixtures/otlp/hand/adk_cycle_poisons_batch.truth.json +++ b/tests/fixtures/otlp/hand/adk_cycle_poisons_batch.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/adk_cycle_poisons_batch", - "source": "hand authored from verify_20261006/s4.py case 'adk 2-cycle' (RecursionError at 94785bb)", + "source": "hand authored; an ADK 2-cycle crash shape (RecursionError at 94785bb)", "extra_turns_allowed": true, "turns": [ { @@ -20,7 +20,7 @@ ], "notes": [ "the healthy trace must survive; turns of the broken trace carry no content and are not compared", - "spec gap D-03: inference under a non agentic root in a trace that also has agentic anchors" + "spec gap: inference under a non agentic root in a trace that also has agentic anchors" ], "content_captured": true, "tokens": { diff --git a/tests/fixtures/otlp/hand/adk_go_main.truth.json b/tests/fixtures/otlp/hand/adk_go_main.truth.json index 4df46d31..c304881b 100644 --- a/tests/fixtures/otlp/hand/adk_go_main.truth.json +++ b/tests/fixtures/otlp/hand/adk_go_main.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/adk_go_main", - "source": "hand authored from reports/C_adk_kagent_telemetry.md section 2.1 (adk-go main 967bfab, SPAN_ONLY)", + "source": "hand authored (adk-go main 967bfab, SPAN_ONLY)", "turns": [ { "user_text": "What is the weather in Paris?", diff --git a/tests/fixtures/otlp/hand/openai_agents_handoff.truth.json b/tests/fixtures/otlp/hand/openai_agents_handoff.truth.json index bc7ea3f2..9f629065 100644 --- a/tests/fixtures/otlp/hand/openai_agents_handoff.truth.json +++ b/tests/fixtures/otlp/hand/openai_agents_handoff.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/openai_agents_handoff", - "source": "hand authored from reports/E_landscape.md (genai-openai-agents 1.2b0 handoff = invoke_workflow > 2x invoke_agent)", + "source": "hand authored (genai-openai-agents 1.2b0 handoff = invoke_workflow > 2x invoke_agent)", "turns": [ { "user_text": "I need a refund for order 42.", diff --git a/tests/fixtures/otlp/hand/self_parent.truth.json b/tests/fixtures/otlp/hand/self_parent.truth.json index 77d2c5bd..f35b7566 100644 --- a/tests/fixtures/otlp/hand/self_parent.truth.json +++ b/tests/fixtures/otlp/hand/self_parent.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/self_parent", - "source": "hand authored; an inference span names itself as parent (crash shape from verify_20261006/s4.py)", + "source": "hand authored; an inference span names itself as parent (a crash shape)", "turns": [ { "user_text": "Ping?", diff --git a/tests/fixtures/otlp/hand/strands_span_events.truth.json b/tests/fixtures/otlp/hand/strands_span_events.truth.json index 7348b7d7..31c0753c 100644 --- a/tests/fixtures/otlp/hand/strands_span_events.truth.json +++ b/tests/fixtures/otlp/hand/strands_span_events.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/strands_span_events", - "source": "hand authored from reports/E_part3a_dialects.md (Strands gen_ai_latest_experimental, span events). Validate against a live run at STOP 4.", + "source": "hand authored (Strands gen_ai_latest_experimental, span events)", "turns": [ { "user_text": "What is the weather in Paris?", diff --git a/tests/fixtures/otlp/hand/two_span_cycle.truth.json b/tests/fixtures/otlp/hand/two_span_cycle.truth.json index 458d6d4a..b5a40199 100644 --- a/tests/fixtures/otlp/hand/two_span_cycle.truth.json +++ b/tests/fixtures/otlp/hand/two_span_cycle.truth.json @@ -1,7 +1,7 @@ { "schema": "otel_core.truth/v1", "fixture": "hand/two_span_cycle", - "source": "hand authored; two inference spans parent each other (crash shape from verify_20261006/s4.py)", + "source": "hand authored; two inference spans parent each other (a crash shape)", "turns": [ { "user_text": "Ping?", @@ -29,7 +29,7 @@ } ], "notes": [ - "the cycle is broken deterministically (M1 rule: earliest start becomes the root); the pair collapses to one logical call" + "the cycle is broken deterministically (earliest start becomes the root); the pair collapses to one logical call" ], "content_captured": true, "tokens": { diff --git a/tests/fixtures/otlp/hand/upstream_new_parts.truth.json b/tests/fixtures/otlp/hand/upstream_new_parts.truth.json index 5c6903d0..8495a5bf 100644 --- a/tests/fixtures/otlp/hand/upstream_new_parts.truth.json +++ b/tests/fixtures/otlp/hand/upstream_new_parts.truth.json @@ -16,7 +16,7 @@ } ], "notes": [ - "server_tool_call parts run at the provider; they are kept in messages but are not client tool trajectory (decision D-02 in deviations.md)", + "server_tool_call parts run at the provider; they are kept in messages but are not client tool trajectory", "reasoning and compaction parts are not user or final text" ], "content_captured": true, diff --git a/tests/fixtures/otlp/samples/tempo_export_with_batches.truth.json b/tests/fixtures/otlp/samples/tempo_export_with_batches.truth.json index d518f9ea..e5be5933 100644 --- a/tests/fixtures/otlp/samples/tempo_export_with_batches.truth.json +++ b/tests/fixtures/otlp/samples/tempo_export_with_batches.truth.json @@ -39,6 +39,6 @@ "output": 129 }, "notes": [ - "Tempo batches export; three nested inference levels (call_llm > generate_content > openai.chat) with equal usage; the controller side client invoke_agent (helm-agent) is under a different root (the export lacks the linking spans) and has nothing inside it, so it yields no turn (D-17)" + "Tempo batches export; three nested inference levels (call_llm > generate_content > openai.chat) with equal usage; the controller side client invoke_agent (helm-agent) is under a different root (the export lacks the linking spans) and has nothing inside it, so it yields no turn" ] } \ No newline at end of file diff --git a/tests/genai/test_continuations.py b/tests/genai/test_continuations.py index e54679f6..e248b5c7 100644 --- a/tests/genai/test_continuations.py +++ b/tests/genai/test_continuations.py @@ -56,8 +56,8 @@ def test_tool_round_across_traces_is_one_turn(): def test_without_content_the_turns_stay_apart_with_a_warning(): conversation = extract_conversation(_traces(*_tool_round(content=False)), "s") assert len(conversation.turns) == 2 - assert any("may continue" in w for w in conversation.turns[1].warnings) - assert not any("may continue" in w for w in conversation.turns[0].warnings) + assert any("may continue" in w.reason for w in conversation.turns[1].warnings) + assert not any("may continue" in w.reason for w in conversation.turns[0].warnings) def test_a_new_question_is_not_joined(): diff --git a/tests/genai/test_corpus.py b/tests/genai/test_corpus.py index b5e342df..fe711f3d 100644 --- a/tests/genai/test_corpus.py +++ b/tests/genai/test_corpus.py @@ -65,7 +65,7 @@ def _turns(conversation): "intermediate": [_text(x) for x in (text_of([m]) for m in t.intermediate_outputs) if x], "agents": sorted(t.agents), "tools": [(c.name, _json(c.arguments), _json(c.result), bool(c.is_error)) for c in t.tool_calls], - "tokens": (t.usage.input_tokens, t.usage.output_tokens), + "tokens": (t.usage.input_tokens or 0, t.usage.output_tokens or 0), "llm_calls": len(t.llm_calls), "evaluations": len(t.external_evaluations), } @@ -105,8 +105,8 @@ def test_file_extraction_matches_truth(truth_path): g = {**g, optional: None} assert g == w, f"turn {i + 1}" total = ( - sum(t.usage.input_tokens for t in conversation.turns), - sum(t.usage.output_tokens for t in conversation.turns), + sum(t.usage.input_tokens or 0 for t in conversation.turns), + sum(t.usage.output_tokens or 0 for t in conversation.turns), ) assert total == (truth["tokens"]["input"], truth["tokens"]["output"]) diff --git a/tests/genai/test_failed_turns.py b/tests/genai/test_failed_turns.py index b9fcc80e..b64d141c 100644 --- a/tests/genai/test_failed_turns.py +++ b/tests/genai/test_failed_turns.py @@ -2,11 +2,13 @@ from __future__ import annotations +from opentelemetry.proto.trace.v1 import trace_pb2 + +from agentevals.api.views import turn_view from agentevals.genai import semconv as sc from agentevals.genai.extract import analyze, extract_conversation from agentevals.otel.decode import decode_json_document from agentevals.otel.model import build_traces -from agentevals.streaming.manager import invocation_dict from otel.builders import agent, chat, request, sid, span STATUS_ERROR = 2 @@ -63,11 +65,11 @@ def test_failed_agent_with_a_model_call_keeps_the_call(): assert turn.status == "error" and len(turn.llm_calls) == 1 -def test_session_payload_reports_status_and_content_capture(): +def test_turn_view_reports_status_and_content_capture(): [turn] = extract_conversation(_traces(_failed_agent(error_type="runtime_error"))).turns - payload = invocation_dict(turn, {}) + view = turn_view(turn, {}) - assert payload["status"] == "error" - assert payload["errorType"] == "runtime_error" - assert payload["contentCaptured"] is False - assert payload["warnings"] == [] + assert view.status == trace_pb2.Status.STATUS_CODE_ERROR + assert view.error_type == "runtime_error" + assert view.content_captured is False + assert list(view.gaps) == [] diff --git a/tests/genai/test_tools.py b/tests/genai/test_tools.py index a3a3f8d3..c0cd9344 100644 --- a/tests/genai/test_tools.py +++ b/tests/genai/test_tools.py @@ -100,7 +100,7 @@ def test_a_tool_call_without_a_span_keeps_its_place_in_the_trajectory(self): ("second", {"n": 2}), ("third", {"n": 3}), ] - assert any("second appears only in message parts" in w for w in turn.warnings) + assert any("second appears only in message parts" in w.reason for w in turn.warnings) def test_a_result_already_in_a_calls_input_does_not_answer_its_new_request(self): call = chat( @@ -114,7 +114,7 @@ def test_a_result_already_in_a_calls_input_does_not_answer_its_new_request(self) ) turn = _turn(call) assert [(t.call_id, t.result) for t in turn.tool_calls] == [("call_2", None)] - assert not [w for w in turn.warnings if "by order" in w] + assert not [w for w in turn.warnings if "by order" in w.reason] def test_results_still_pair_by_order_across_calls(self): first = chat( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d19ad8a7..e17fe40c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -305,7 +305,7 @@ async def wait_for_session_complete( session_id: str, timeout: float = 5.0, ) -> None: - """Poll until the session is complete and its ``session_complete`` was sent, or raise TimeoutError.""" + """Poll until the session is complete and its complete view was published, or raise TimeoutError.""" deadline = asyncio.get_event_loop().time() + timeout while asyncio.get_event_loop().time() < deadline: session = mgr.sessions.get(session_id) diff --git a/tests/integration/test_live_agents.py b/tests/integration/test_live_agents.py index cec7bf38..db97aae4 100644 --- a/tests/integration/test_live_agents.py +++ b/tests/integration/test_live_agents.py @@ -92,7 +92,7 @@ def test_session_created_with_spans_and_logs(self, live_servers): assert session.span_count > 0, "Expected spans from LLM calls" assert session.log_count > 0, "LangChain uses logs for message content" - def test_invocations_extracted_with_content(self, live_servers): + def test_turns_extracted_with_content(self, live_servers): main_port, otlp_http_port, mgr = live_servers session_name = "e2e-langchain-inv" @@ -106,10 +106,11 @@ def test_invocations_extracted_with_content(self, live_servers): wait_for_session_complete_sync(mgr, session_name, timeout=30) session = mgr.sessions[session_name] - assert len(session.invocations) > 0, "Expected extracted invocations" - for inv in session.invocations: - has_content = inv.get("userText") or inv.get("agentResponse") - assert has_content, f"Invocation {inv.get('invocationId', '?')} has no content" + turns = mgr.view(session).with_turns.upsert.turns + assert len(turns) > 0, "Expected extracted turns" + for turn in turns: + has_content = turn.HasField("user_text") or turn.HasField("answer_text") + assert has_content, f"Turn {turn.span_id} has no content" def test_session_visible_via_api(self, live_servers): main_port, otlp_http_port, mgr = live_servers @@ -154,7 +155,7 @@ def test_session_created_spans_only(self, live_servers): assert session.is_complete assert session.span_count > 0, "Expected spans from LLM calls" - def test_invocations_extracted(self, live_servers): + def test_turns_extracted(self, live_servers): main_port, otlp_http_port, mgr = live_servers session_name = "e2e-strands-inv" @@ -171,7 +172,7 @@ def test_invocations_extracted(self, live_servers): wait_for_session_complete_sync(mgr, session_name, timeout=30) session = mgr.sessions[session_name] - assert len(session.invocations) > 0, "Expected extracted invocations" + assert len(mgr.view(session).with_turns.upsert.turns) > 0, "Expected extracted turns" def test_session_visible_via_api(self, live_servers): main_port, otlp_http_port, mgr = live_servers @@ -216,7 +217,7 @@ def test_session_created_spans_only(self, live_servers): assert session.is_complete assert session.span_count > 0, "Expected spans from ADK agent" - def test_invocations_extracted(self, live_servers): + def test_turns_extracted(self, live_servers): main_port, otlp_http_port, mgr = live_servers session_name = "e2e-adk-inv" @@ -230,7 +231,7 @@ def test_invocations_extracted(self, live_servers): wait_for_session_complete_sync(mgr, session_name, timeout=30) session = mgr.sessions[session_name] - assert len(session.invocations) > 0, "Expected extracted invocations" + assert len(mgr.view(session).with_turns.upsert.turns) > 0, "Expected extracted turns" def test_session_visible_via_api(self, live_servers): main_port, otlp_http_port, mgr = live_servers @@ -272,7 +273,7 @@ def test_session_created_with_spans(self, live_servers): assert session.is_complete assert session.span_count > 0, "Expected spans from LLM calls" - def test_invocations_extracted(self, live_servers): + def test_turns_extracted(self, live_servers): main_port, otlp_http_port, mgr = live_servers session_name = "e2e-openai-agents-inv" @@ -286,7 +287,7 @@ def test_invocations_extracted(self, live_servers): wait_for_session_complete_sync(mgr, session_name, timeout=30) session = mgr.sessions[session_name] - assert len(session.invocations) > 0, "Expected extracted invocations" + assert len(mgr.view(session).with_turns.upsert.turns) > 0, "Expected extracted turns" def test_session_visible_via_api(self, live_servers): main_port, otlp_http_port, mgr = live_servers @@ -329,7 +330,7 @@ def test_session_created_with_spans(self, live_servers): assert session.is_complete assert session.span_count > 0, "Expected spans from LLM calls" - def test_invocations_extracted_with_content(self, live_servers): + def test_turns_extracted_with_content(self, live_servers): main_port, otlp_http_port, mgr = live_servers session_name = "e2e-pydantic-ai-inv" @@ -343,10 +344,11 @@ def test_invocations_extracted_with_content(self, live_servers): wait_for_session_complete_sync(mgr, session_name, timeout=30) session = mgr.sessions[session_name] - assert len(session.invocations) > 0, "Expected extracted invocations" - for inv in session.invocations: - has_content = inv.get("userText") or inv.get("agentResponse") - assert has_content, f"Invocation {inv.get('invocationId', '?')} has no content" + turns = mgr.view(session).with_turns.upsert.turns + assert len(turns) > 0, "Expected extracted turns" + for turn in turns: + has_content = turn.HasField("user_text") or turn.HasField("answer_text") + assert has_content, f"Turn {turn.span_id} has no content" def test_session_visible_via_api(self, live_servers): main_port, otlp_http_port, mgr = live_servers diff --git a/tests/otel/test_own_spans.py b/tests/otel/test_own_spans.py new file mode 100644 index 00000000..58c72ccd --- /dev/null +++ b/tests/otel/test_own_spans.py @@ -0,0 +1,47 @@ +"""agentevals' own API server spans never become live sessions, even when exported back to it.""" + +from __future__ import annotations + +from agentevals.otel.identity import GRPC_INSTRUMENTATION_SCOPE +from agentevals.otel.store import DROPPED_OWN_SPANS + +from .builders import ingest, make_store, named, request, span + +SERVER = 2 +CLIENT = 3 + + +def _rpc_span(name: str, kind: int) -> dict: + data = span(name=name, attrs={"rpc.system.name": "grpc"}) + data["kind"] = kind + return data + + +def _ingest(name: str, kind: int = SERVER, scope: str = GRPC_INSTRUMENTATION_SCOPE): + store, _ = make_store() + result = ingest(store, request([_rpc_span(name, kind)], named("own"), scope=scope)) + return store, result + + +def test_api_server_spans_are_dropped(): + store, result = _ingest("/agentevals.api.v1alpha1.SessionService/WatchSessions") + assert (result.accepted, result.rejected) == (1, 0) + assert result.dropped[DROPPED_OWN_SPANS] == 1 + assert store.counters[DROPPED_OWN_SPANS] == 1 + assert store.sessions == {} + + +def test_other_grpc_server_spans_are_kept(): + store, result = _ingest("/acme.orders.v1.OrderService/Create") + assert result.dropped[DROPPED_OWN_SPANS] == 0 + assert list(store.sessions) == ["own"] + + +def test_client_spans_of_the_api_are_kept(): + store, _ = _ingest("/agentevals.api.v1alpha1.SessionService/ListSessions", kind=CLIENT) + assert list(store.sessions) == ["own"] + + +def test_the_scope_must_be_the_grpc_instrumentation(): + store, _ = _ingest("/agentevals.api.v1alpha1.SessionService/ListSessions", scope="acme.grpc") + assert list(store.sessions) == ["own"] diff --git a/tests/resolvers/test_kubernetes.py b/tests/resolvers/test_kubernetes.py index b2e3581a..e035c19a 100644 --- a/tests/resolvers/test_kubernetes.py +++ b/tests/resolvers/test_kubernetes.py @@ -40,7 +40,7 @@ async def test_missing_fields_raise(self): with pytest.raises(ValueError, match="namespace, name, key"): await resolver.resolve({"kind": "kubernetes"}) - async def test_key_not_found_lists_names_not_values(self): + async def test_key_not_found_names_neither_keys_nor_values(self): secret_value = _b64("sk-do-not-leak") client = _client_returning({"api-key": secret_value, "tls.crt": _b64("cert")}) resolver = KubernetesSecretResolver(client) @@ -49,8 +49,9 @@ async def test_key_not_found_lists_names_not_values(self): await resolver.resolve({"namespace": "ns", "name": "creds", "key": "wrong"}) message = str(exc.value) - assert "api-key" in message and "tls.crt" in message - # The enumeration must never echo the base64-encoded secret values. + assert "'wrong' not found in Secret ns/creds" in message + # The error reaches API callers, so it must not reveal what else the Secret holds. + assert "api-key" not in message and "tls.crt" not in message assert secret_value not in message diff --git a/tests/run/test_fetcher.py b/tests/run/test_fetcher.py index 833975f9..4adf0f42 100644 --- a/tests/run/test_fetcher.py +++ b/tests/run/test_fetcher.py @@ -6,7 +6,7 @@ import pytest -from agentevals.run.fetcher import HttpTraceFetcher, InlineTraceFetcher, resolve_fetcher +from agentevals.run.fetcher import HttpTraceFetcher, InlineTraceFetcher, resolve_fetcher, validate_trace_id from agentevals.storage.models import TraceTarget @@ -77,3 +77,38 @@ async def test_missing_trace_id_raises(self): fetcher = HttpTraceFetcher() with pytest.raises(ValueError, match="base_url"): await fetcher.fetch(TraceTarget(kind="http"), context={}) + + +class TestValidateTraceId: + @pytest.mark.parametrize("trace_id", ["1dbd779ff94bf24c", "1dbd779ff94bf24c7b84678a5dc0346c", "1DBD779FF94BF24C"]) + def test_accepts_8_and_16_byte_hex_ids(self, trace_id): + assert validate_trace_id(trace_id) == trace_id + + @pytest.mark.parametrize( + "trace_id", + [ + "", + "1dbd779ff94bf24", + "1dbd779ff94bf24c7", + "1dbd779ff94bf24c7b84678a5dc0346c0", + "../../admin/export", + "1dbd779ff94bf24c/../x", + "1dbd779ff94bf24c%2F", + "1dbd779ff94bf24c\n", + "1dbd779ff94bf24g", + ], + ) + def test_rejects_anything_else(self, trace_id): + with pytest.raises(ValueError, match="trace_id"): + validate_trace_id(trace_id) + + async def test_fetcher_refuses_before_any_request(self, monkeypatch): + import agentevals.run.fetcher as fetcher_module + + def no_client(*args, **kwargs): + raise AssertionError("no request may be made") + + monkeypatch.setattr(fetcher_module.httpx, "AsyncClient", no_client) + target = TraceTarget(kind="http", base_url="https://traces.example.com/api", trace_id="../admin") + with pytest.raises(ValueError, match="trace_id"): + await HttpTraceFetcher().fetch(target, context={}) diff --git a/tests/run/test_result_builder.py b/tests/run/test_result_builder.py index d0300a26..7284de57 100644 --- a/tests/run/test_result_builder.py +++ b/tests/run/test_result_builder.py @@ -65,7 +65,7 @@ def test_multiple_metrics_flatten(self): assert names == ["a", "a", "b", "c"] def test_eval_set_item_id_defaults_to_trace_id(self): - """OSS scope: no per-eval-case id extraction. Trace id is the stable + """No per-eval-case id extraction. Trace id is the stable identifier for both eval_set_item_id and eval_set_item_name. Test locks this so future changes are deliberate.""" rr = RunResult(trace_results=[_trace_result(_mr())]) diff --git a/tests/run/test_service.py b/tests/run/test_service.py index 5acd19be..8ff5b98f 100644 --- a/tests/run/test_service.py +++ b/tests/run/test_service.py @@ -79,7 +79,7 @@ async def test_cancel_unknown_run_returns_false(self, service): class TestRecordEvalRun: - """Option A: /api/evaluate synchronously persists runs + results.""" + """/api/evaluate synchronously persists runs + results.""" def _params(self) -> EvalParams: return EvalParams(evaluators=[BuiltinMetricDef(name="m1")]) diff --git a/tests/run/test_sinks.py b/tests/run/test_sinks.py index 73a6a65f..fdad17ea 100644 --- a/tests/run/test_sinks.py +++ b/tests/run/test_sinks.py @@ -425,3 +425,41 @@ async def emit_error(self, run_id, error, attempt): await fanout.emit_final(run_id, {}, attempt=1) await fanout.emit_error(run_id, "x", attempt=1) assert good_writes == ["partial", "final", "error"] + + +class TestWebhookLogs: + """Webhook URLs and response bodies often carry tokens; logs keep the origin and the status.""" + + MARK = "qxmark" + + async def _emit(self, url: str, handler) -> None: + with _mock_async_client(httpx.MockTransport(handler)): + await HttpWebhookSink(url, max_attempts=1).emit_final(uuid4(), {}, attempt=1) + + async def test_rejection(self, caplog): + caplog.set_level(logging.WARNING, logger="agentevals.run.sinks") + await self._emit( + f"https://hooks.example.com/services/{self.MARK}?sig={self.MARK}", + lambda request: httpx.Response(401, text=f"bad token {self.MARK}"), + ) + assert "https://hooks.example.com returned 401" in caplog.text + assert self.MARK not in caplog.text + + async def test_server_error(self, caplog): + caplog.set_level(logging.WARNING, logger="agentevals.run.sinks") + await self._emit( + f"https://user:{self.MARK}@hooks.example.com/x", + lambda request: httpx.Response(503, text=self.MARK), + ) + assert "https://hooks.example.com failed after 1 attempts: HTTP 503" in caplog.text + assert self.MARK not in caplog.text + + async def test_transport_error(self, caplog): + caplog.set_level(logging.WARNING, logger="agentevals.run.sinks") + + def refuse(request): + raise httpx.ConnectError(f"connect to {self.MARK} refused") + + await self._emit(f"https://hooks.example.com/{self.MARK}", refuse) + assert "failed after 1 attempts: ConnectError" in caplog.text + assert self.MARK not in caplog.text diff --git a/tests/storage/test_memory_repos.py b/tests/storage/test_memory_repos.py index cbb15648..fef9c8a6 100644 --- a/tests/storage/test_memory_repos.py +++ b/tests/storage/test_memory_repos.py @@ -195,7 +195,7 @@ async def test_delete_by_run(self, repos): class TestSessionRepository: - """SessionRepository is forward-compat scaffolding in this slice; cover + """SessionRepository is forward-compat scaffolding; cover the basic CRUD surface so regressions surface if the protocol drifts.""" async def test_upsert_and_get(self, repos): diff --git a/tests/storage/test_postgres_repos.py b/tests/storage/test_postgres_repos.py new file mode 100644 index 00000000..6cd60f28 --- /dev/null +++ b/tests/storage/test_postgres_repos.py @@ -0,0 +1,136 @@ +"""Postgres run and result repositories against a live database. + +Runs only when AGENTEVALS_TEST_DATABASE_URL points at a disposable Postgres; each test gets a fresh +schema, dropped afterwards. +""" + +from __future__ import annotations + +import os +from datetime import timedelta +from uuid import UUID, uuid4 + +import pytest + +from agentevals.storage.models import Result, ResultStatus, Run, RunSpec, RunStatus, TraceTarget +from agentevals.storage.postgres.migrator import Migrator +from agentevals.storage.repos.postgres import PostgresRepos + +pytestmark = pytest.mark.skipif( + not os.environ.get("AGENTEVALS_TEST_DATABASE_URL"), + reason="requires AGENTEVALS_TEST_DATABASE_URL pointing at a disposable Postgres", +) + +SCHEMA = "agentevals_test_repos" +LEASE = timedelta(seconds=30) +MAX_ATTEMPTS = 3 +EXPIRED = timedelta(minutes=-1) + + +@pytest.fixture +async def db(): + import asyncpg + + dsn = os.environ["AGENTEVALS_TEST_DATABASE_URL"] + pool = await asyncpg.create_pool(dsn, min_size=1, max_size=4) + await pool.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA}" CASCADE') + await Migrator(dsn=dsn, schema=SCHEMA, lock_timeout_s=10).up() + try: + yield PostgresRepos(pool=pool, schema=SCHEMA), pool + finally: + await pool.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA}" CASCADE') + await pool.close() + + +async def _queued(repos: PostgresRepos) -> UUID: + run = Run(run_id=uuid4(), status=RunStatus.QUEUED, spec=RunSpec(target=TraceTarget(kind="inline", inline={}))) + return (await repos.runs.create(run)).run_id + + +async def _running(pool, run_id: UUID, *, attempt: int, lease_left: timedelta, cancel: bool = False) -> None: + await pool.execute( + f""" + UPDATE "{SCHEMA}".run + SET status = 'running', worker_id = 'gone-worker', claimed_at = now() - interval '5 minutes', + started_at = now() - interval '5 minutes', lease_expires_at = now() + $2, + attempt = $3, cancel_requested = $4 + WHERE run_id = $1 + """, + run_id, + lease_left, + attempt, + cancel, + ) + + +async def _claim(repos: PostgresRepos) -> Run | None: + return await repos.runs.claim_next(worker_id="new-worker", lease=LEASE, max_attempts=MAX_ATTEMPTS) + + +class TestResults: + async def test_unscored_invocations_keep_their_position(self, db): + repos, _ = db + run_id = await _queued(repos) + result = Result( + result_id="r1", + run_id=run_id, + eval_set_item_id="item", + eval_set_item_name="trace", + evaluator_name="tool_trajectory_avg_score", + evaluator_type="builtin", + status=ResultStatus.PASSED, + score=0.75, + per_invocation_scores=[0.5, None, 1.0], + ) + await repos.results.upsert_many(run_id, [result]) + [stored] = await repos.results.list_by_run(run_id) + assert stored.per_invocation_scores == [0.5, None, 1.0] + + await repos.results.upsert_many(run_id, [result.model_copy(update={"per_invocation_scores": [None]})]) + [updated] = await repos.results.list_by_run(run_id) + assert updated.per_invocation_scores == [None] + + +class TestExpiredLeases: + async def test_the_last_attempt_fails_the_run(self, db): + repos, pool = db + run_id = await _queued(repos) + await _running(pool, run_id, attempt=MAX_ATTEMPTS, lease_left=EXPIRED) + assert await _claim(repos) is None + run = await repos.runs.get(run_id) + assert (run.status, run.error) == (RunStatus.FAILED, "worker lease expired on the last attempt") + assert run.finished_at is not None + + async def test_a_cancel_request_cancels_the_run(self, db): + repos, pool = db + run_id = await _queued(repos) + await _running(pool, run_id, attempt=1, lease_left=EXPIRED, cancel=True) + assert await _claim(repos) is None + run = await repos.runs.get(run_id) + assert (run.status, run.error) == (RunStatus.CANCELLED, None) + assert run.finished_at is not None + + async def test_a_run_with_attempts_left_is_claimed_again(self, db): + repos, pool = db + run_id = await _queued(repos) + await _running(pool, run_id, attempt=1, lease_left=EXPIRED) + claimed = await _claim(repos) + assert (claimed.run_id, claimed.attempt, claimed.worker_id) == (run_id, 2, "new-worker") + assert claimed.status == RunStatus.RUNNING + + async def test_a_live_lease_is_left_alone(self, db): + repos, pool = db + run_id = await _queued(repos) + await _running(pool, run_id, attempt=MAX_ATTEMPTS, lease_left=-EXPIRED) + assert await _claim(repos) is None + run = await repos.runs.get(run_id) + assert (run.status, run.worker_id, run.finished_at) == (RunStatus.RUNNING, "gone-worker", None) + + async def test_finished_runs_are_never_touched(self, db): + repos, pool = db + run_id = await _queued(repos) + await _running(pool, run_id, attempt=MAX_ATTEMPTS, lease_left=EXPIRED) + await repos.runs.update_status(run_id, RunStatus.SUCCEEDED, summary={"passed": 1}) + before = await repos.runs.get(run_id) + assert await _claim(repos) is None + assert await repos.runs.get(run_id) == before diff --git a/tests/streaming/test_live.py b/tests/streaming/test_live.py index e1b709e7..3574b8c1 100644 --- a/tests/streaming/test_live.py +++ b/tests/streaming/test_live.py @@ -1,99 +1,28 @@ -"""Live manager: the append only UI diff, broadcast back pressure and keeping the event loop free.""" +"""Live manager: session views of running sessions and keeping the event loop free.""" from __future__ import annotations import asyncio +from datetime import UTC, datetime import pytest +from agentevals.api.v1alpha1 import views_pb2 +from agentevals.api.views import SessionFacts, session_view from agentevals.genai.extract import extract_conversation -from agentevals.genai.model import Conversation, LlmCall, Turn, Usage +from agentevals.genai.model import Conversation from agentevals.otel.decode import decode_json_document -from agentevals.otel.model import SpanRef, build_traces +from agentevals.otel.model import build_traces from agentevals.streaming import manager as manager_module -from agentevals.streaming.manager import LiveManager, LiveState, live_events -from otel.builders import agent, assistant, chat, named, request, sid, span, tid, tool_result, user +from agentevals.streaming.manager import LiveManager +from otel.builders import agent, assistant, chat, named, request, span, user -def _conversation(*spans: dict) -> Conversation: +def _conversation(*spans: dict) -> tuple[Conversation, dict]: decoded = decode_json_document(request(list(spans)), strict=True) traces, _ = build_traces(decoded.spans, decoded.logs) - return extract_conversation(traces, "s") - - -def _kinds(events: list[dict]) -> list[str]: - return [e["type"] for e in events] - - -class TestLiveEvents: - def test_a_late_agent_span_does_not_repeat_elements(self): - state = LiveState() - call = chat(span_id="c1", parent="a1", inputs=[user("Roll a die")], outputs=[assistant("4")]) - first = live_events("s", _conversation(call), state, incomplete={tid("t1")}) - assert _kinds(first) == ["user_input", "agent_response", "token_update"] - - second = live_events("s", _conversation(agent(span_id="a1"), call), state, incomplete={tid("t1")}) - assert second == [] - - def test_a_late_wrapper_span_does_not_repeat_the_response(self): - state = LiveState() - leaf = chat(span_id="p1", parent="w1", outputs=[assistant("4")]) - live_events("s", _conversation(leaf), state, incomplete=set()) - wrapper = chat(span_id="w1", outputs=[assistant("4")]) - assert "agent_response" not in _kinds(live_events("s", _conversation(wrapper, leaf), state, incomplete=set())) - - def test_tool_call_and_result_are_sent_once(self): - state = LiveState() - spans = [ - chat( - span_id="c1", - inputs=[user("roll")], - outputs=[assistant(tool_calls=[("call_1", "roll_die", {"sides": 6})])], - ), - ] - assert _kinds(live_events("s", _conversation(*spans), state, set())).count("tool_call") == 1 - spans.append( - chat( - span_id="c2", - start=3_000, - end=4_000, - inputs=[ - user("roll"), - assistant(tool_calls=[("call_1", "roll_die", {"sides": 6})]), - tool_result("call_1", "roll_die", 5), - ], - outputs=[assistant("You rolled 5")], - ) - ) - kinds = _kinds(live_events("s", _conversation(*spans), state, set())) - assert kinds.count("tool_call") == 0 - assert kinds.count("tool_result") == 1 - - def test_invocation_id_is_the_trace_until_the_trace_completes(self): - conversation = _conversation(agent(span_id="a1"), chat(span_id="c1", parent="a1")) - open_ = live_events("s", conversation, LiveState(), incomplete={tid("t1")}) - done = live_events("s", conversation, LiveState(), incomplete=set()) - assert {e["invocationId"] for e in open_} == {tid("t1")} - assert {e["invocationId"] for e in done} == {sid("a1")} - - def test_token_updates_are_signed_deltas(self): - ref = SpanRef(tid("t1"), sid("c1")) - call = LlmCall(ref=ref, operation="chat", usage=Usage(input_tokens=8, output_tokens=2)) - turn = Turn(ref=ref, llm_calls=[call], usage=Usage(input_tokens=8, output_tokens=2)) - state = LiveState(input_tokens=30, output_tokens=2) - events = live_events("s", Conversation(key="s", turns=[turn]), state, set()) - update = next(e for e in events if e["type"] == "token_update") - assert (update["inputTokens"], update["outputTokens"]) == (-22, 0) - assert (state.input_tokens, state.output_tokens) == (8, 2) - - def test_elements_beyond_the_identity_cap_are_left_to_session_complete(self, monkeypatch): - monkeypatch.setattr(manager_module, "MAX_EMITTED_KEYS", 2) - spans = [ - chat(trace=f"t{n}", span_id=f"c{n}", outputs=[assistant(f"answer {n}")], start=n * 10, end=n * 10 + 5) - for n in range(5) - ] - events = live_events("s", _conversation(*spans), LiveState(), set()) - assert _kinds(events).count("agent_response") + _kinds(events).count("user_input") <= 2 + spans_by_id = {(s.trace_id, s.span_id): s for t in traces for s in t.spans.values()} + return extract_conversation(traces, "s"), spans_by_id def _tool_layer(span_id: str, parent: str, start: int, **attrs) -> dict: @@ -114,13 +43,16 @@ class TestToolLayers: pytest.param([CLIENT, SERVER], [FRAMEWORK], id="framework-last"), ], ) - def test_layers_of_one_call_arriving_apart_are_sent_once(self, early, late): - state = LiveState() + def test_layers_of_one_call_arriving_apart_are_one_call(self, early, late): root = agent(span_id="a1", start=0, end=10_000) - sent = live_events("s", _conversation(root, *early), state, set()) - sent += live_events("s", _conversation(root, *early, *late), state, set()) - assert _kinds(sent).count("tool_call") == 1 - assert _kinds(sent).count("tool_result") == 1 + facts = SessionFacts( + session_id="s", received_at=datetime.now(UTC), is_complete=False, trace_ids=(), span_count=0 + ) + partial = session_view(facts, *_conversation(root, *early)) + assert sum(len(t.tool_calls) for t in partial.turns) <= 1 + turn = session_view(facts, *_conversation(root, *early, *late)).turns[0] + assert [t.name for t in turn.tool_calls] == ["search"] + assert turn.tool_calls[0].HasField("result") class TestSessionList: @@ -132,64 +64,13 @@ async def test_a_running_session_lists_its_turns(self): ) await mgr.ingest_spans(decode_json_document(request([waiting], named("open")), strict=True)) for _ in range(100): - if mgr.sessions["open"].invocations: - break - await asyncio.sleep(0.02) - info = mgr.session_info(mgr.sessions["open"], with_invocations=True) - await mgr.shutdown() - assert info.is_complete is False - assert [i["userText"] for i in info.invocations] == ["still there?"] - - -async def _drain_until(client, kind: str, timeout: float = 10.0) -> list[dict]: - events: list[dict] = [] - - async def _read(): - while True: - event = await client.queue.get() - if event is None: - return - events.append(event) - if event.get("type") == kind: - return - - await asyncio.wait_for(_read(), timeout) - return events - - -def _many_calls(n: int) -> dict: - spans = [agent(span_id="a1", start=1, end=10**9)] - spans += [ - chat(span_id=f"c{i}", parent="a1", outputs=[assistant(f"step {i}")], start=1_000 + i, end=1_500 + i) - for i in range(n) - ] - return request(spans, named("burst")) - - -class TestBroadcast: - async def test_a_large_recompute_reaches_a_reading_client(self): - mgr = LiveManager(completion_grace_seconds=0.0) - mgr.start() - client = mgr.register_sse_client() - reader = asyncio.create_task(_drain_until(client, "session_complete")) - await mgr.ingest_spans(decode_json_document(_many_calls(1_500), strict=True)) - events = await reader - await mgr.shutdown() - assert client.dropped is False - assert _kinds(events).count("agent_response") == 1_500 - - async def test_a_client_that_does_not_read_is_dropped(self): - mgr = LiveManager(completion_grace_seconds=0.0) - mgr.start() - client = mgr.register_sse_client() - await mgr.ingest_spans(decode_json_document(_many_calls(1_500), strict=True)) - for _ in range(200): - if client.dropped: + session = mgr.view(mgr.sessions["open"]).with_turns.upsert + if session.turns: break await asyncio.sleep(0.02) await mgr.shutdown() - assert client.dropped is True - assert client not in mgr.clients + assert session.summary.state == views_pb2.SESSION_STATE_LIVE + assert [t.user_text for t in session.turns] == ["still there?"] class TestEventLoop: diff --git a/tests/streaming/test_watchers.py b/tests/streaming/test_watchers.py new file mode 100644 index 00000000..88706512 --- /dev/null +++ b/tests/streaming/test_watchers.py @@ -0,0 +1,122 @@ +"""Session watchers: the latest pending event per session, a snapshot on subscribe, and a cap.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from agentevals.api.v1alpha1 import sessions_pb2 +from agentevals.otel.decode import decode_json_document +from agentevals.otel.store import Limits +from agentevals.streaming import manager as manager_module +from agentevals.streaming.manager import LiveManager, TooManyWatchers, Watcher, WatchEvent +from otel.builders import chat, named, request + + +def _event(session_id: str, span_count: int = 0) -> WatchEvent: + session = sessions_pb2.WatchSessionsResponse().upsert + session.summary.id = session_id + session.summary.span_count = span_count + return WatchEvent.upsert(session) + + +async def _drain(watcher: Watcher) -> list[sessions_pb2.WatchSessionsResponse]: + """Everything the watcher holds, in order. Closes it first, so this never waits.""" + watcher.close() + out = [] + while (event := await watcher.next()) is not None: + out.append(event.with_turns) + return out + + +def _decoded(name: str): + return decode_json_document(request([chat(trace=name)], named(name)), strict=True) + + +class TestWatcher: + async def test_a_newer_event_replaces_the_pending_one_and_moves_last(self): + watcher = Watcher(include_turns=True) + watcher.put("a", _event("a", 1)) + watcher.put("b", _event("b", 1)) + watcher.put("a", _event("a", 2)) + events = await _drain(watcher) + assert [(e.upsert.summary.id, e.upsert.summary.span_count) for e in events] == [("b", 1), ("a", 2)] + + async def test_next_waits_for_an_event(self): + watcher = Watcher(include_turns=True) + waiting = asyncio.create_task(watcher.next()) + await asyncio.sleep(0.01) + assert not waiting.done() + watcher.put("a", _event("a")) + assert (await asyncio.wait_for(waiting, 1)).with_turns.upsert.summary.id == "a" + + async def test_a_closed_watcher_hands_out_what_it_holds_then_ends(self): + watcher = Watcher(include_turns=True) + watcher.put("a", _event("a")) + watcher.close() + assert (await watcher.next()).with_turns.upsert.summary.id == "a" + assert await watcher.next() is None + + async def test_close_wakes_a_waiting_reader(self): + watcher = Watcher(include_turns=True) + waiting = asyncio.create_task(watcher.next()) + await asyncio.sleep(0.01) + watcher.close() + assert await asyncio.wait_for(waiting, 1) is None + + def test_summary_watchers_get_the_summary_form(self): + session = sessions_pb2.WatchSessionsResponse().upsert + session.summary.id = "a" + session.turns.add(index=0) + event = WatchEvent.upsert(session) + assert len(Watcher(include_turns=True).response(event).upsert.turns) == 1 + summary_only = Watcher(include_turns=False).response(event) + assert (summary_only.upsert.summary.id, len(summary_only.upsert.turns)) == ("a", 0) + + +class TestSubscribe: + async def test_snapshot_is_newest_first_then_synced(self): + mgr = LiveManager() + for name in ("first", "second"): + await mgr.ingest_spans(_decoded(name)) + events = await _drain(mgr.subscribe(include_turns=False)) + kinds = [(e.WhichOneof("event"), e.upsert.summary.id) for e in events] + assert kinds == [("upsert", "second"), ("upsert", "first"), ("synced", "")] + + def test_watchers_are_capped(self, monkeypatch): + monkeypatch.setattr(manager_module, "MAX_WATCHERS", 2) + mgr = LiveManager() + first = mgr.subscribe(include_turns=False) + mgr.subscribe(include_turns=False) + with pytest.raises(TooManyWatchers): + mgr.subscribe(include_turns=False) + mgr.unsubscribe(first) + assert first.closed + mgr.subscribe(include_turns=False) + + def test_unsubscribe_twice_is_harmless(self): + mgr = LiveManager() + watcher = mgr.subscribe(include_turns=False) + mgr.unsubscribe(watcher) + mgr.unsubscribe(watcher) + assert mgr.watchers == [] + + async def test_a_removed_session_replaces_its_pending_upsert(self): + mgr = LiveManager(Limits(max_sessions=1)) + watcher = mgr.subscribe(include_turns=False) + decoded = _decoded("old") + mgr.load_session("old", decoded.spans, decoded.logs) + await mgr.ingest_spans(_decoded("new")) + assert list(mgr.sessions) == ["new"] + events = await _drain(watcher) + assert [e.WhichOneof("event") for e in events if e.upsert.summary.id == "old" or e.removed.id == "old"] == [ + "removed" + ] + assert "new" in [e.upsert.summary.id for e in events] + + async def test_shutdown_closes_every_watcher(self): + mgr = LiveManager() + watchers = [mgr.subscribe(include_turns=False) for _ in range(3)] + await mgr.shutdown() + assert mgr.watchers == [] and all(w.closed for w in watchers) diff --git a/tests/test_api.py b/tests/test_api.py index dd712b9e..7acb6254 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -63,12 +63,11 @@ def _reset(mgr) -> None: mgr.store.traces.clear() -def _add_session(mgr, session_id="sess1", is_complete=True, spans=None, invocations=None, eval_set_id=None): +def _add_session(mgr, session_id="sess1", is_complete=True, spans=None, eval_set_id=None): """Seed a live session the way a bug report replay does (no routing, no recompute).""" decoded = decode_bare_spans_json(spans or [], strict=False) session = mgr.store.load_session(session_id, decoded.spans, [], eval_set_id=eval_set_id) session.is_complete = is_complete - session.invocations = invocations or [] return session @@ -448,11 +447,6 @@ class TestMetricsEndpoint: def setup_class(cls): cls.client = TestClient(_make_app()) - def test_metrics_fallback(self): - with patch.dict("sys.modules", {"google.adk.evaluation.metric_evaluator_registry": None}): - body = _assert_envelope(self.client.get("/api/metrics")) - assert len(body["data"]) == 11 - def test_metrics_envelope(self): body = _assert_envelope(self.client.get("/api/metrics")) assert isinstance(body["data"], list) @@ -766,7 +760,7 @@ def test_stream_done_event(self, mock_load_traces, mock_eval): assert "result" in done assert "traceResults" in done["result"] - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.load_traces") def test_stream_resolves_credential_refs(self, mock_load_traces, mock_eval, monkeypatch): monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-stream") @@ -794,7 +788,7 @@ def test_stream_bad_credential_refs(self): assert '"error"' in resp.text assert "credentialRefs" in resp.text - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.load_eval_set") @patch("agentevals.api.routes.load_traces") def test_stream_sanitizes_traversal_filenames(self, mock_load_traces, mock_load_eval_set, mock_eval): @@ -866,7 +860,7 @@ class TestEvaluateJson: def setup_class(cls): cls.client = TestClient(_make_app()) - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_success(self, mock_eval): mock_eval.return_value = _make_run_result() resp = self.client.post( @@ -880,7 +874,7 @@ def test_evaluate_json_success(self, mock_eval): assert "traceResults" in body["data"] assert len(body["data"]["traceResults"]) == 1 - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_camel_case_config(self, mock_eval): mock_eval.return_value = _make_run_result() resp = self.client.post( @@ -921,7 +915,7 @@ def test_evaluate_json_empty_traces(self): assert resp.status_code == 400 assert "No traces" in resp.json()["detail"] - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_with_eval_set(self, mock_eval): mock_eval.return_value = _make_run_result() resp = self.client.post( @@ -981,7 +975,7 @@ def test_evaluate_json_rejects_legacy_metrics_field(self): ) assert resp.status_code == 422 - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_camel_keys_in_result(self, mock_eval): mock_eval.return_value = _make_run_result() resp = self.client.post( @@ -994,7 +988,7 @@ def test_evaluate_json_camel_keys_in_result(self, mock_eval): body = resp.json() _assert_all_keys_camel(body) - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_default_config(self, mock_eval): mock_eval.return_value = _make_run_result() resp = self.client.post( @@ -1004,7 +998,7 @@ def test_evaluate_json_default_config(self, mock_eval): body = _assert_envelope(resp) assert "traceResults" in body["data"] - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_resolves_credential_refs(self, mock_eval, monkeypatch): monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-json") captured: dict = {} @@ -1020,7 +1014,7 @@ def test_evaluate_json_resolves_credential_refs(self, mock_eval, monkeypatch): _assert_envelope(resp) assert captured["judge_key"] == "sk-resolved-json" - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_without_credential_refs_is_noop(self, mock_eval): captured: dict = {} mock_eval.side_effect = _capturing_run_eval(captured) @@ -1038,7 +1032,7 @@ def test_evaluate_json_credential_refs_wrong_shape_returns_422(self): ) assert resp.status_code == 422 - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) def test_evaluate_json_unresolvable_credential_returns_400(self, mock_eval, monkeypatch): monkeypatch.delenv("AE_MISSING_KEY", raising=False) mock_eval.return_value = _make_run_result() @@ -1065,7 +1059,7 @@ class TestEvaluateJsonStream: def setup_class(cls): cls.client = TestClient(_make_app()) - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.OtlpJsonLoader") def test_stream_content_type(self, mock_loader_cls, mock_eval): mock_loader_cls.return_value.load_from_dict.return_value = [] @@ -1076,7 +1070,7 @@ def test_stream_content_type(self, mock_loader_cls, mock_eval): ) assert resp.headers["content-type"].startswith("text/event-stream") - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.OtlpJsonLoader") def test_stream_done_event(self, mock_loader_cls, mock_eval): mock_trace = MagicMock() @@ -1114,7 +1108,7 @@ def test_stream_error_on_empty_traces(self): assert '"error"' in body assert "No traces" in body - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.OtlpJsonLoader") def test_stream_resolves_credential_refs(self, mock_loader_cls, mock_eval, monkeypatch): monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-json-stream") @@ -1134,7 +1128,7 @@ def test_stream_resolves_credential_refs(self, mock_loader_cls, mock_eval, monke assert '"done"' in resp.text assert captured["judge_key"] == "sk-resolved-json-stream" - @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock) + @patch("agentevals.api.evaluation.run_evaluation_from_traces", new_callable=AsyncMock) @patch("agentevals.api.routes.OtlpJsonLoader") def test_stream_unresolvable_credential_yields_error(self, mock_loader_cls, mock_eval, monkeypatch): monkeypatch.delenv("AE_MISSING_KEY", raising=False) @@ -1184,14 +1178,6 @@ def test_list_sessions_with_data(self): ids = {s["sessionId"] for s in body["data"]} assert ids == {"s1", "s2"} - def test_list_sessions_complete_includes_invocations(self): - _reset(self.mgr) - invs = [{"invocation_id": "inv1", "user_content": "hello"}] - _add_session(self.mgr, "s1", is_complete=True, invocations=invs) - client = TestClient(self.app) - body = _assert_envelope(client.get("/api/streaming/sessions")) - assert body["data"][0]["invocations"] is not None - # --------------------------------------------------------------------------- # POST /api/streaming/create-eval-set @@ -1751,19 +1737,16 @@ class TestUIUpdatesSSE: """ def _make_streaming_app(self): - import asyncio - from types import SimpleNamespace - from agentevals.api.app import create_app - from agentevals.streaming.manager import LiveManager + from agentevals.streaming.manager import LiveManager, Watcher mgr = LiveManager() - # Replace register_sse_client so the queue immediately closes (None sentinel) - # so the streaming response can be read synchronously in tests. - q: asyncio.Queue = asyncio.Queue() - q.put_nowait(None) - mgr.register_sse_client = MagicMock(return_value=SimpleNamespace(queue=q)) - mgr.unregister_sse_client = MagicMock() + # A watcher that is already closed ends the stream at once, so the response can be read + # synchronously in tests. + watcher = Watcher(include_turns=True) + watcher.close() + mgr.subscribe = MagicMock(return_value=watcher) + mgr.unsubscribe = MagicMock() return create_app(enable_streaming=True, trace_manager=mgr) def test_sse_endpoint_returns_200_not_422(self): diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index 3e410070..1f214a18 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -487,7 +487,7 @@ def test_resource_attributes_as_flat_dict(self): class TestNestedDictAttributes: - """Tests for ClickHouse JSON column format (nested dicts auto-flattened).""" + """Tests for JSON column exports (nested dicts auto-flattened).""" def test_nested_dict_flattened_to_dot_notation(self): loader = OtlpJsonLoader() diff --git a/tests/test_otlp_receiver.py b/tests/test_otlp_receiver.py index c3b7a8a5..e73103a5 100644 --- a/tests/test_otlp_receiver.py +++ b/tests/test_otlp_receiver.py @@ -33,6 +33,7 @@ ingest_logs, ingest_traces, ) +from agentevals.api.v1alpha1 import views_pb2 from agentevals.cli import _install_shared_exit_handler from agentevals.genai.routing import GenAIRoutingPolicy from agentevals.otel.encode import encode_document @@ -800,15 +801,25 @@ async def test_capacity_refusal_of_every_item_is_overload(self): # --------------------------------------------------------------------------- -# Live manager: SSE and recompute +# Live manager: watchers and recompute # --------------------------------------------------------------------------- -def _drain(client) -> list[dict]: +async def _drain(watcher) -> list: + """Every pending ``WatchSessionsResponse`` (with turns), without waiting for new ones.""" out = [] - while not client.queue.empty(): - out.append(client.queue.get_nowait()) - return out + while True: + try: + event = await asyncio.wait_for(watcher.next(), 0.05) + except TimeoutError: + return out + if event is None: + return out + out.append(event.with_turns) + + +def _upserts(events: list) -> list: + return [e.upsert for e in events if e.HasField("upsert")] async def _settle(mgr: LiveManager) -> None: @@ -820,65 +831,61 @@ async def _settle(mgr: LiveManager) -> None: class TestLiveManager: - async def test_ingest_broadcasts_session_started_and_span_stubs(self): + async def test_ingest_publishes_the_new_session(self): mgr = LiveManager() - client = mgr.register_sse_client() + watcher = mgr.subscribe(include_turns=True) await mgr.ingest_spans(decode_traces_json(_request([_chat()], _named("s1", "e1")))) - events = _drain(client) - started = next(e for e in events if e["type"] == "session_started") - assert (started["session"]["sessionId"], started["session"]["evalSetId"]) == ("s1", "e1") - received = next(e for e in events if e["type"] == "span_received") - assert received["span"] == {"traceId": tid("t1"), "spanId": sid("c1"), "parentSpanId": None, "name": "chat gpt"} - assert "attributes" not in received["span"] - - async def test_recompute_streams_elements_then_session_complete(self): + events = await _drain(watcher) + assert events[0].HasField("synced") + summary = _upserts(events)[0].summary + assert (summary.id, summary.eval_set_id) == ("s1", "e1") + + async def test_recompute_publishes_turns_then_the_complete_session(self): mgr = LiveManager(completion_grace_seconds=0.0) - client = mgr.register_sse_client() + watcher = mgr.subscribe(include_turns=True) await mgr.ingest_spans(decode_traces_json(_request([_chat(text="Roll", reply="Rolled 4")], _named("s1")))) await asyncio.sleep(0.01) await _settle(mgr) - events = _drain(client) - kinds = [e["type"] for e in events] - assert kinds.index("user_input") < kinds.index("session_complete") - complete = next(e for e in events if e["type"] == "session_complete") - assert complete["invocations"][0]["userText"] == "Roll" - assert complete["invocations"][0]["agentText"] == "Rolled 4" - assert complete["invocations"][0]["invocationId"] == sid("c1") - assert sum(e["inputTokens"] for e in events if e["type"] == "token_update") == 10 - assert mgr.sessions["s1"].invocations == complete["invocations"] - - async def test_reopened_session_sends_only_session_complete(self): + session = _upserts(await _drain(watcher))[-1] + assert session.summary.state == views_pb2.SESSION_STATE_COMPLETE + turn = session.turns[0] + assert (turn.user_text, turn.answer_text, turn.span_id) == ("Roll", "Rolled 4", sid("c1")) + assert session.summary.totals.usage.input_tokens == 10 + assert mgr.view(mgr.sessions["s1"]).with_turns.upsert == session + + async def test_reopened_session_is_published_complete_again(self): mgr = LiveManager(completion_grace_seconds=0.0) - client = mgr.register_sse_client() + watcher = mgr.subscribe(include_turns=True) await mgr.ingest_spans(decode_traces_json(_request([_chat(span="c1")], _named("s1")))) await asyncio.sleep(0.01) await _settle(mgr) - _drain(client) + await _drain(watcher) await mgr.ingest_spans(decode_traces_json(_request([_chat(trace="t1", span="c2", text="Again")], _named("s1")))) await asyncio.sleep(0.01) await _settle(mgr) - kinds = {e["type"] for e in _drain(client)} - assert kinds <= {"span_received", "session_complete"} - assert "session_complete" in kinds + session = _upserts(await _drain(watcher))[-1] + assert session.summary.state == views_pb2.SESSION_STATE_COMPLETE + assert "Again" in [t.user_text for t in session.turns] - async def test_slow_sse_client_is_dropped(self): - mgr = LiveManager() - client = mgr.register_sse_client() - for n in range(1001): - mgr.broadcast({"type": "noise", "n": n}) - assert client.dropped is True - assert client not in mgr.clients - assert client.queue.get_nowait() is None + async def test_a_watcher_that_does_not_read_keeps_one_event_per_session(self): + mgr = LiveManager(completion_grace_seconds=0.0) + watcher = mgr.subscribe(include_turns=False) + for n in range(20): + await mgr.ingest_spans(decode_traces_json(_request([_chat(span=f"c{n}")], _named("s1")))) + await _settle(mgr) + events = await _drain(watcher) + assert [e.WhichOneof("event") for e in events] == ["synced", "upsert"] + assert watcher in mgr.watchers async def test_debug_load_session_is_complete_and_listed(self): mgr = LiveManager() - client = mgr.register_sse_client() + watcher = mgr.subscribe(include_turns=True) decoded = decode_traces_json(_request([_chat()])) session = mgr.load_session("bundle", decoded.spans, [], eval_set_id="e") await _settle(mgr) assert session.is_complete and session.session_id == "bundle" - kinds = [e["type"] for e in _drain(client)] - assert kinds == ["session_started", "session_complete"] + events = await _drain(watcher) + assert _upserts(events)[-1].summary.state == views_pb2.SESSION_STATE_COMPLETE async def test_shutdown_stops_the_ticker(self): mgr = LiveManager() diff --git a/ui/.gitignore b/ui/.gitignore index a547bf36..602cbead 100644 --- a/ui/.gitignore +++ b/ui/.gitignore @@ -1,5 +1,5 @@ # Logs -logs +/logs *.log npm-debug.log* yarn-debug.log* diff --git a/ui/eslint.config.js b/ui/eslint.config.js index 5e6b472f..4cf82f9b 100644 --- a/ui/eslint.config.js +++ b/ui/eslint.config.js @@ -6,7 +6,7 @@ import tseslint from 'typescript-eslint' import { defineConfig, globalIgnores } from 'eslint/config' export default defineConfig([ - globalIgnores(['dist']), + globalIgnores(['dist', 'src/generated']), { files: ['**/*.{ts,tsx}'], extends: [ diff --git a/ui/package-lock.json b/ui/package-lock.json index 8be57de3..fdc65011 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,6 +8,7 @@ "name": "ui", "version": "0.0.0", "dependencies": { + "@bufbuild/protobuf": "^2.16.0", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "antd": "^6.6.5", @@ -426,6 +427,12 @@ "node": ">=6.9.0" } }, + "node_modules/@bufbuild/protobuf": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz", + "integrity": "sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, "node_modules/@cacheable/memory": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", diff --git a/ui/package.json b/ui/package.json index 381460ea..88858540 100644 --- a/ui/package.json +++ b/ui/package.json @@ -10,6 +10,7 @@ "preview": "vite preview" }, "dependencies": { + "@bufbuild/protobuf": "^2.16.0", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "antd": "^6.6.5", diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 4931ebf9..21e2c668 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -63,52 +63,6 @@ export async function convertTraces( return unwrap(response); } -export async function evaluateTracesAPI( - traceFiles: File[], - evalSetFile: File | null, - evalConfig: EvalConfig -): Promise { - const formData = new FormData(); - - traceFiles.forEach(file => { - formData.append('trace_files', file); - }); - - if (evalSetFile) { - formData.append('eval_set_file', evalSetFile); - } - - formData.append('config', JSON.stringify(evalConfig)); - - try { - const response = await fetch(`${API_BASE_URL}/evaluate`, { - method: 'POST', - body: formData, - }); - - if (!response.ok) { - let errorMessage = `API error: ${response.statusText}`; - try { - const errorData = await response.json(); - if (errorData.detail) { - errorMessage = errorData.detail; - } - } catch { - // Fallback to statusText if JSON parsing fails - } - throw new Error(errorMessage); - } - - return unwrap(response); - - } catch (error) { - if (error instanceof Error) { - throw error; - } - throw new Error('Unknown error occurred while evaluating traces'); - } -} - export async function evaluateTracesStreaming( traceFiles: File[], evalSetFile: File | null, @@ -257,27 +211,6 @@ export async function listMetrics(): Promise { } } -export async function validateEvalSet(evalSetFile: File): Promise<{ valid: boolean; evalSetId?: string; numCases?: number; errors?: string[] }> { - const formData = new FormData(); - formData.append('eval_set_file', evalSetFile); - - try { - const response = await fetch(`${API_BASE_URL}/validate/eval-set`, { - method: 'POST', - body: formData, - }); - - if (!response.ok) { - throw new Error(`Failed to validate eval set: ${response.statusText}`); - } - - return unwrap<{ valid: boolean; evalSetId?: string; numCases?: number; errors?: string[] }>(response); - } catch (error) { - console.error('Failed to validate eval set:', error); - throw error; - } -} - export async function getConfig(): Promise<{ apiKeys: { google: boolean; anthropic: boolean; openai: boolean }; otlp?: { httpPort: number; grpcPort: number } | null; diff --git a/ui/src/components/annotation-queue/AnnotationDetailPanel.tsx b/ui/src/components/annotation-queue/AnnotationDetailPanel.tsx index 55af2c22..0892c874 100644 --- a/ui/src/components/annotation-queue/AnnotationDetailPanel.tsx +++ b/ui/src/components/annotation-queue/AnnotationDetailPanel.tsx @@ -1,6 +1,6 @@ import { useState } from 'react'; import type { AnnotationQueueItem, Annotation, FirstPassLabel } from '../../lib/types'; -import { ToolCallMessage, ToolResultMessage } from '../streaming/LiveMessage'; +import { TurnMessages } from '../streaming/LiveConversationPanel'; interface AnnotationDetailPanelProps { item: AnnotationQueueItem; @@ -25,11 +25,8 @@ export function AnnotationDetailPanel({ item, onSave, onClose }: AnnotationDetai setTimeout(() => setSaved(false), 2000); }; - const invocations = item.invocations || []; - const elements = item.liveElements || []; - - const totalTokens = item.totalTokens || - (item.liveStats ? item.liveStats.totalInputTokens + item.liveStats.totalOutputTokens : 0); + const turns = item.session.turns; + const totalTokens = item.totalTokens ?? 0; return (
@@ -116,15 +113,15 @@ export function AnnotationDetailPanel({ item, onSave, onClose }: AnnotationDetai Conversation
- {invocations.length > 0 ? ( + {turns.length > 0 ? (
- {invocations.map((inv, idx) => ( + {turns.map((turn, idx) => (
Turn {idx + 1}
- -
-
User
-
- {inv.userText || '(no text)'} -
-
- - {inv.toolCalls.length > 0 && ( -
-
Tool Calls
- {inv.toolCalls.map((tc, i) => { - const tr = inv.toolResponses?.find(r => r.id === tc.id || r.name === tc.name); - return ( -
- - {tr && } -
- ); - })} -
- )} - - {inv.agentText && ( -
-
Agent
-
- {inv.agentText} -
-
- )} -
- ))} -
- ) : elements.length > 0 ? ( -
- {elements.map((el, idx) => ( -
- {el.type === 'tool_call' ? ( - - ) : el.type === 'tool_result' ? ( - - ) : ( - <> -
- {el.type === 'user_input' ? 'User' : 'Agent'} -
-
- {el.data?.text || '(no text)'} -
- - )} +
))}
diff --git a/ui/src/components/annotation-queue/AnnotationQueueView.tsx b/ui/src/components/annotation-queue/AnnotationQueueView.tsx index a5a15280..2270c1bf 100644 --- a/ui/src/components/annotation-queue/AnnotationQueueView.tsx +++ b/ui/src/components/annotation-queue/AnnotationQueueView.tsx @@ -62,7 +62,7 @@ export function AnnotationQueueView() { const traceFiles = await Promise.all( currentQueue.items.map(async (item) => { - if (!item.invocations || item.invocations.length === 0) return null; + if (item.session.turns.length === 0) return null; const traceResponse = await fetch(config.api.endpoints.streamingGetTrace, { method: 'POST', @@ -100,7 +100,7 @@ export function AnnotationQueueView() { } }; - const completedItems = currentQueue?.items.filter(item => item.invocations && item.invocations.length > 0) || []; + const completedItems = currentQueue?.items.filter(item => item.session.turns.length > 0) || []; return (
= ({ title: 'Conversation', key: 'conversation', width: 280, - render: (_: any, record: AnnotationQueueItem) => { - const invocations = record.invocations || []; - const elements = record.liveElements || []; - const numTurns = invocations.length || (elements.filter(e => e.type === 'user_input').length) || 0; + render: (_: unknown, record: AnnotationQueueItem) => { + const numTurns = record.session.turns.length; const isExpanded = expandedRowKeys.includes(record.sessionId); - - const firstUserInput = invocations[0]?.userText || - elements.find(e => e.type === 'user_input')?.data?.text || ''; + const firstUserInput = record.session.summary?.firstUserText ?? ''; if (numTurns > 1) { return ( @@ -226,7 +223,7 @@ export const AnnotationTable: React.FC = ({ title: 'Annotated', key: 'annotated', width: 110, - render: (_: any, record: AnnotationQueueItem) => { + render: (_: unknown, record: AnnotationQueueItem) => { if (!record.annotation) { return ( — @@ -243,10 +240,9 @@ export const AnnotationTable: React.FC = ({ title: 'EvalSet', key: 'evalset', width: 130, - render: (_: any, record: AnnotationQueueItem) => { + render: (_: unknown, record: AnnotationQueueItem) => { const isGolden = selectedGoldenId === record.sessionId; - const hasInvocations = record.invocations && record.invocations.length > 0; - if (!hasInvocations) return —; + if (record.session.turns.length === 0) return —; return ( - {session.status === 'complete' && onRemove && ( + {complete && onRemove && (
diff --git a/ui/src/components/streaming/SessionMetadata.tsx b/ui/src/components/streaming/SessionMetadata.tsx index 64aa0738..3549cd72 100644 --- a/ui/src/components/streaming/SessionMetadata.tsx +++ b/ui/src/components/streaming/SessionMetadata.tsx @@ -1,22 +1,9 @@ +import type { SessionSummary } from '../../generated/agentevals/api/v1alpha1/views_pb'; +import { tokens } from '../../lib/liveSessions'; + interface SessionMetadataProps { - session: { - sessionId: string; - traceId: string; - metadata: Record; - startedAt: string; - status: 'active' | 'complete'; - invocations?: Array<{ - modelInfo?: { - provider?: string; - cacheCreationTokens?: number; - cacheReadTokens?: number; - }; - }>; - }; - liveStats: { - totalInputTokens: number; - totalOutputTokens: number; - }; + summary: SessionSummary; + provider?: string; } function MetadataItem({ label, children }: { label: string; children: React.ReactNode }) { @@ -42,11 +29,10 @@ function MetadataItem({ label, children }: { label: string; children: React.Reac ); } -export function SessionMetadata({ session, liveStats }: SessionMetadataProps) { - const totalTokens = liveStats.totalInputTokens + liveStats.totalOutputTokens; - const provider = session.invocations?.[0]?.modelInfo?.provider; - const totalCacheCreation = session.invocations?.reduce((sum, inv) => sum + (inv.modelInfo?.cacheCreationTokens || 0), 0) || 0; - const totalCacheRead = session.invocations?.reduce((sum, inv) => sum + (inv.modelInfo?.cacheReadTokens || 0), 0) || 0; +export function SessionMetadata({ summary, provider }: SessionMetadataProps) { + const { input, output, cacheRead = 0, cacheWrite = 0 } = tokens(summary.totals?.usage); + const total = (input ?? 0) + (output ?? 0); + const totals = summary.totals; return (
- {totalTokens > 0 && ( + {summary.agent && {summary.agent}} + + {summary.agentVersion && {summary.agentVersion}} + + {total > 0 && ( - {totalTokens.toLocaleString()} + {total.toLocaleString()} - (↓{liveStats.totalInputTokens.toLocaleString()} ↑{liveStats.totalOutputTokens.toLocaleString()}) + (↓{(input ?? 0).toLocaleString()} ↑{(output ?? 0).toLocaleString()}) + {totals && totals.modelCallsWithUsage < totals.modelCalls && ( + + measured in {totals.modelCallsWithUsage} of {totals.modelCalls} calls + + )} )} - {provider && ( - {provider} - )} + {provider && {provider}} - {(totalCacheCreation > 0 || totalCacheRead > 0) && ( + {(cacheWrite > 0 || cacheRead > 0) && ( - {totalCacheRead > 0 && `${totalCacheRead.toLocaleString()} read`} - {totalCacheCreation > 0 && totalCacheRead > 0 && ' / '} - {totalCacheCreation > 0 && `${totalCacheCreation.toLocaleString()} created`} + {cacheRead > 0 && `${cacheRead.toLocaleString()} read`} + {cacheWrite > 0 && cacheRead > 0 && ' / '} + {cacheWrite > 0 && `${cacheWrite.toLocaleString()} created`} )} - {Object.keys(session.metadata).length > 0 && Object.entries(session.metadata).map(([key, value]) => ( - {String(value)} - ))} + {summary.conversationId && {summary.conversationId}} + + {summary.spanCount.toLocaleString()}
); } diff --git a/ui/src/components/streaming/turnStatus.ts b/ui/src/components/streaming/turnStatus.ts deleted file mode 100644 index 44986bd2..00000000 --- a/ui/src/components/streaming/turnStatus.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { StreamingInvocation } from '../../lib/types'; - -type ToolCall = StreamingInvocation['toolCalls'][number]; -type ToolResponse = NonNullable[number]; - -export function isFailedTurn(inv: Pick): boolean { - return inv.status === 'error' || !!inv.errorType; -} - -/** True only when the server reported every turn as captured without message content. */ -export function lacksContent(invocations: StreamingInvocation[] | undefined): boolean { - return !!invocations?.length && invocations.every(inv => inv.contentCaptured === false); -} - -/** - * The response for each call, or undefined. Ids decide when both sides have one; otherwise the - * next unused response with the same name, so repeated calls to one tool keep their own results. - */ -export function pairToolResponses(calls: ToolCall[], responses: ToolResponse[]): Array { - const used = new Set(); - const take = (match: (r: ToolResponse) => boolean) => { - const i = responses.findIndex((r, j) => !used.has(j) && match(r)); - if (i < 0) return undefined; - used.add(i); - return responses[i]; - }; - return calls.map(tc => - (tc.id ? take(r => r.id === tc.id) : undefined) - ?? take(r => r.name === tc.name && !(tc.id && r.id)), - ); -} diff --git a/ui/src/components/upload/EvalSetEditorDrawer.tsx b/ui/src/components/upload/EvalSetEditorDrawer.tsx index 3eeafe8b..e3181d6a 100644 --- a/ui/src/components/upload/EvalSetEditorDrawer.tsx +++ b/ui/src/components/upload/EvalSetEditorDrawer.tsx @@ -9,14 +9,21 @@ import { JsonPreview } from '../builder/JsonPreview'; import { readFileAsText, convertSnakeToCamel, convertCamelToSnake } from '../../lib/utils'; import type { EvalSet, EvalSetMetadata, EvalCase, Invocation } from '../../lib/types'; -function parseEvalSetFromJson(raw: any): EvalSet { +interface RawEvalSet { + eval_set_id?: string; + name?: string; + description?: string; + eval_cases?: { eval_id?: string; conversation?: unknown[] }[]; +} + +function parseEvalSetFromJson(raw: RawEvalSet): EvalSet { return { eval_set_id: raw.eval_set_id || '', name: raw.name || '', description: raw.description || '', - eval_cases: (raw.eval_cases || []).map((ec: any) => ({ + eval_cases: (raw.eval_cases || []).map(ec => ({ eval_id: ec.eval_id || '', - conversation: (ec.conversation || []).map((inv: any) => convertSnakeToCamel(inv) as Invocation), + conversation: (ec.conversation || []).map(inv => convertSnakeToCamel(inv) as Invocation), })), }; } @@ -47,16 +54,23 @@ export const EvalSetEditorDrawer: React.FC = ({ file, const [dirty, setDirty] = useState(false); const [error, setError] = useState(null); + // Opening the drawer, or another file, starts over (state adjusted during render, not in an effect). + const [shown, setShown] = useState({ open, file }); + if (shown.open !== open || shown.file !== file) { + setShown({ open, file }); + if (open) { + setLoading(true); + setError(null); + setDirty(false); + } + } + useEffect(() => { if (!open) return; - setLoading(true); - setError(null); - setDirty(false); - readFileAsText(file) .then((content) => { - const parsed = JSON.parse(content); + const parsed = JSON.parse(content) as RawEvalSet; setEvalSet(parseEvalSetFromJson(parsed)); setLoading(false); }) diff --git a/ui/src/components/upload/TraceEditorDrawer.tsx b/ui/src/components/upload/TraceEditorDrawer.tsx index 2a6182a1..4f40a8eb 100644 --- a/ui/src/components/upload/TraceEditorDrawer.tsx +++ b/ui/src/components/upload/TraceEditorDrawer.tsx @@ -32,13 +32,20 @@ export const TraceEditorDrawer: React.FC = ({ file, file const [dirty, setDirty] = useState(false); const [error, setError] = useState(null); + // Opening the drawer, or another file, starts over (state adjusted during render, not in an effect). + const [shown, setShown] = useState({ open, file }); + if (shown.open !== open || shown.file !== file) { + setShown({ open, file }); + if (open) { + setLoading(true); + setError(null); + setDirty(false); + } + } + useEffect(() => { if (!open) return; - setLoading(true); - setError(null); - setDirty(false); - Promise.all([ readFileAsText(file).then(async (content) => { const parsed = parseTraceFileForEditing(content, file.name); @@ -111,16 +118,13 @@ export const TraceEditorDrawer: React.FC = ({ file, file return applyEditsAndSerialize(parsedFile, allInvocations, editMappings); }, [parsedFile, traceGroups, editMappings, dirty]); - const [initialContent, setInitialContent] = useState(''); - useEffect(() => { - if (open && parsedFile) { - if (parsedFile.format === 'otlp-jsonl') { - setInitialContent(parsedFile.rawData.map((line: any) => JSON.stringify(line)).join('\n')); - } else { - setInitialContent(JSON.stringify(parsedFile.rawData, null, 2)); - } + const initialContent = useMemo(() => { + if (!parsedFile) return ''; + if (parsedFile.format === 'otlp-jsonl') { + return (parsedFile.rawData as unknown[]).map(line => JSON.stringify(line)).join('\n'); } - }, [open, parsedFile]); + return JSON.stringify(parsedFile.rawData, null, 2); + }, [parsedFile]); const totalInvocations = traceGroups.reduce((sum, g) => sum + g.invocations.length, 0); diff --git a/ui/src/config.ts b/ui/src/config.ts index 3a35ded7..ae65ce69 100644 --- a/ui/src/config.ts +++ b/ui/src/config.ts @@ -5,15 +5,9 @@ export const config = { api: { baseUrl: API_BASE_URL, endpoints: { - health: `${API_BASE_URL}/api/health`, - metrics: `${API_BASE_URL}/api/metrics`, - evaluate: `${API_BASE_URL}/api/evaluate`, - evaluateStream: `${API_BASE_URL}/api/evaluate/stream`, runs: `${API_BASE_URL}/api/runs`, - validateEvalSet: `${API_BASE_URL}/api/validate/eval-set`, streamingCreateEvalSet: `${API_BASE_URL}/api/streaming/create-eval-set`, streamingGetTrace: `${API_BASE_URL}/api/streaming/get-trace`, - streamingSessions: `${API_BASE_URL}/api/streaming/sessions`, uiUpdatesStream: `${API_BASE_URL}/stream/ui-updates`, debugBundle: `${API_BASE_URL}/api/debug/bundle`, debugLoad: `${API_BASE_URL}/api/debug/load`, diff --git a/ui/src/context/TraceContext.tsx b/ui/src/context/TraceContext.tsx index aeb52b23..8a2bc695 100644 --- a/ui/src/context/TraceContext.tsx +++ b/ui/src/context/TraceContext.tsx @@ -1,5 +1,6 @@ import { createContext, useContext } from 'react'; -import type { TraceResult, ViewType, EvalSet, EvalSetMetadata, EvalCase, TraceTableRow, LiveSession, AnnotationQueue, Annotation } from '../lib/types'; +import type { TraceResult, ViewType, EvalSet, EvalSetMetadata, EvalCase, TraceTableRow, AnnotationQueue, Annotation } from '../lib/types'; +import type { Session } from '../lib/liveSessions'; import type { TraceMetadata } from '../lib/trace-metadata'; export interface ApiKeyStatus { @@ -38,7 +39,7 @@ export interface TraceState { version: string | null; // Streaming state - streamingSessions: Map; + streamingSessions: Map; // Annotation queue state annotationQueues: AnnotationQueue[]; @@ -62,7 +63,7 @@ export interface TraceContextType { runEvaluation: () => Promise; setCurrentView: (view: ViewType) => void; setEvaluationOrigin: (view: ViewType | null) => void; - setStreamingSessions: (updater: (prev: Map) => Map) => void; + setStreamingSessions: (updater: (prev: Map) => Map) => void; removeSession: (sessionId: string) => void; clearAllSessions: () => void; selectTrace: (traceId: string | null) => void; @@ -71,7 +72,7 @@ export interface TraceContextType { // Annotation queue actions createAnnotationQueue: (name: string) => string; - addToAnnotationQueue: (queueId: string, session: LiveSession) => void; + addToAnnotationQueue: (queueId: string, session: Session) => void; annotateQueueItem: (queueId: string, sessionId: string, annotation: Annotation) => void; setCurrentAnnotationQueueId: (id: string | null) => void; setPendingAnnotations: (annotations: Map) => void; diff --git a/ui/src/context/TraceProvider.tsx b/ui/src/context/TraceProvider.tsx index 6738f6aa..8db4a193 100644 --- a/ui/src/context/TraceProvider.tsx +++ b/ui/src/context/TraceProvider.tsx @@ -2,7 +2,9 @@ import React, { useState, useMemo, useEffect } from 'react'; import type { ReactNode } from 'react'; import { TraceContext } from './TraceContext'; import type { TraceState } from './TraceContext'; -import type { ViewType, EvalConfig, EvalSet, EvalSetMetadata, EvalCase, LiveSession, AnnotationQueue, Annotation } from '../lib/types'; +import type { ViewType, EvalConfig, EvalSet, EvalSetMetadata, EvalCase, AnnotationQueue, Annotation } from '../lib/types'; +import { isComplete, primaryTraceId, sessionId, startedAtIso, totalTokens } from '../lib/liveSessions'; +import type { Session } from '../lib/liveSessions'; import { evaluateTracesStreaming, convertTraces, getConfig, healthCheck } from '../api/client'; import { buildEvalConfig } from '../lib/eval-config'; @@ -277,7 +279,7 @@ export const TraceProvider: React.FC = ({ children }) => { setEvaluationOrigin: (view: ViewType | null) => setState((prev) => ({ ...prev, evaluationOrigin: view })), - setStreamingSessions: (updater: (prev: Map) => Map) => + setStreamingSessions: (updater: (prev: Map) => Map) => setState((prev) => ({ ...prev, streamingSessions: updater(prev.streamingSessions) })), removeSession: (sessionId: string) => @@ -291,7 +293,7 @@ export const TraceProvider: React.FC = ({ children }) => { setState((prev) => ({ ...prev, streamingSessions: new Map( - [...prev.streamingSessions].filter(([, s]) => s.status === 'active') + [...prev.streamingSessions].filter(([, s]) => !isComplete(s)) ), })), @@ -322,25 +324,21 @@ export const TraceProvider: React.FC = ({ children }) => { return id; }, - addToAnnotationQueue: (queueId: string, session: LiveSession) => { + addToAnnotationQueue: (queueId: string, session: Session) => { setState((prev) => { const queues = prev.annotationQueues.map((q) => { + const id = sessionId(session); if (q.id !== queueId) return q; - if (q.items.some((item) => item.sessionId === session.sessionId)) return q; - const totalTokens = - (session.liveStats?.totalInputTokens ?? 0) + - (session.liveStats?.totalOutputTokens ?? 0); + if (q.items.some((item) => item.sessionId === id)) return q; + const tokenCount = totalTokens(session.summary?.totals?.usage); const newItem = { - sessionId: session.sessionId, - traceId: session.traceId, - agentName: session.metadata?.['gen_ai.agent.name'] || session.metadata?.['service.name'], - startTime: session.startedAt, - model: session.liveStats?.model || session.metadata?.model, - totalTokens: totalTokens > 0 ? totalTokens : undefined, - invocations: session.invocations, - liveElements: session.liveElements, - liveStats: session.liveStats, - metadata: session.metadata, + sessionId: id, + traceId: primaryTraceId(session), + agentName: session.summary?.agent || undefined, + startTime: startedAtIso(session), + model: session.summary?.models[0], + totalTokens: tokenCount > 0 ? tokenCount : undefined, + session, }; return { ...q, items: [...q.items, newItem] }; }); diff --git a/ui/src/generated/agentevals/api/v1alpha1/common_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/common_pb.d.ts new file mode 100644 index 00000000..c9c23e84 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/common_pb.d.ts @@ -0,0 +1,212 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/common.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { TracesData } from "../../../opentelemetry/proto/trace/v1/trace_pb"; +import type { LogsData } from "../../../opentelemetry/proto/logs/v1/logs_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/common.proto. + */ +export declare const file_agentevals_api_v1alpha1_common: GenFile; + +/** + * limit 0 means the default of 50; the maximum is 100. + * + * @generated from message agentevals.api.v1alpha1.PageRequest + */ +export declare type PageRequest = Message<"agentevals.api.v1alpha1.PageRequest"> & { + /** + * @generated from field: int32 limit = 1; + */ + limit: number; + + /** + * Opaque token from a previous PageResponse. + * + * @generated from field: string page_token = 2; + */ + pageToken: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.PageRequest. + * Use `create(PageRequestSchema)` to create a new message. + */ +export declare const PageRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.PageResponse + */ +export declare type PageResponse = Message<"agentevals.api.v1alpha1.PageResponse"> & { + /** + * Empty on the last page. + * + * @generated from field: string next_page_token = 1; + */ + nextPageToken: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.PageResponse. + * Use `create(PageResponseSchema)` to create a new message. + */ +export declare const PageResponseSchema: GenMessage; + +/** + * A trace file as bytes, as the REST API takes uploads. At most 10 MiB. + * + * @generated from message agentevals.api.v1alpha1.TraceDocument + */ +export declare type TraceDocument = Message<"agentevals.api.v1alpha1.TraceDocument"> & { + /** + * @generated from field: bytes content = 1; + */ + content: Uint8Array; + + /** + * @generated from field: agentevals.api.v1alpha1.TraceFormat format = 2; + */ + format: TraceFormat; + + /** + * The file name, used in messages and as a session name hint (trace_.json). + * + * @generated from field: string name = 3; + */ + name: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TraceDocument. + * Use `create(TraceDocumentSchema)` to create a new message. + */ +export declare const TraceDocumentSchema: GenMessage; + +/** + * OTLP telemetry as protobuf. Logs carry the GenAI events that belong to the spans. + * + * @generated from message agentevals.api.v1alpha1.OtlpTelemetry + */ +export declare type OtlpTelemetry = Message<"agentevals.api.v1alpha1.OtlpTelemetry"> & { + /** + * @generated from field: opentelemetry.proto.trace.v1.TracesData traces = 1; + */ + traces?: TracesData | undefined; + + /** + * @generated from field: opentelemetry.proto.logs.v1.LogsData logs = 2; + */ + logs?: LogsData | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.OtlpTelemetry. + * Use `create(OtlpTelemetrySchema)` to create a new message. + */ +export declare const OtlpTelemetrySchema: GenMessage; + +/** + * Traces to read. Both fields may be set; their traces are combined. + * + * @generated from message agentevals.api.v1alpha1.TraceInput + */ +export declare type TraceInput = Message<"agentevals.api.v1alpha1.TraceInput"> & { + /** + * @generated from field: agentevals.api.v1alpha1.OtlpTelemetry otlp = 1; + */ + otlp?: OtlpTelemetry | undefined; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.TraceDocument documents = 2; + */ + documents: TraceDocument[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TraceInput. + * Use `create(TraceInputSchema)` to create a new message. + */ +export declare const TraceInputSchema: GenMessage; + +/** + * @generated from enum agentevals.api.v1alpha1.TraceFormat + */ +export enum TraceFormat { + /** + * Detected from the content. + * + * @generated from enum value: TRACE_FORMAT_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * One OTLP/JSON document (hex ids, as the OTLP/HTTP JSON encoding). + * + * @generated from enum value: TRACE_FORMAT_OTLP_JSON = 1; + */ + OTLP_JSON = 1, + + /** + * One OTLP/JSON document or span per line. + * + * @generated from enum value: TRACE_FORMAT_OTLP_JSONL = 2; + */ + OTLP_JSONL = 2, + + /** + * A Jaeger JSON export. + * + * @generated from enum value: TRACE_FORMAT_JAEGER_JSON = 3; + */ + JAEGER_JSON = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.TraceFormat. + */ +export declare const TraceFormatSchema: GenEnum; + +/** + * How traces form evaluation units. + * + * @generated from enum agentevals.api.v1alpha1.GroupBy + */ +export enum GroupBy { + /** + * The default of the field that uses it. + * + * @generated from enum value: GROUP_BY_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * Conversation when any trace carries agentevals.session_name or the eval set has a multi + * turn case, otherwise trace. + * + * @generated from enum value: GROUP_BY_AUTO = 1; + */ + AUTO = 1, + + /** + * Every trace on its own. + * + * @generated from enum value: GROUP_BY_TRACE = 2; + */ + TRACE = 2, + + /** + * Traces sharing agentevals.session_name, gen_ai.conversation.id or session.id form one unit. + * + * @generated from enum value: GROUP_BY_CONVERSATION = 3; + */ + CONVERSATION = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.GroupBy. + */ +export declare const GroupBySchema: GenEnum; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/common_pb.js b/ui/src/generated/agentevals/api/v1alpha1/common_pb.js new file mode 100644 index 00000000..7c2b1272 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/common_pb.js @@ -0,0 +1,75 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/common.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_opentelemetry_proto_logs_v1_logs } from "../../../opentelemetry/proto/logs/v1/logs_pb"; +import { file_opentelemetry_proto_trace_v1_trace } from "../../../opentelemetry/proto/trace/v1/trace_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/common.proto. + */ +export const file_agentevals_api_v1alpha1_common = /*@__PURE__*/ + fileDesc("CiRhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9jb21tb24ucHJvdG8SF2FnZW50ZXZhbHMuYXBpLnYxYWxwaGExIjAKC1BhZ2VSZXF1ZXN0Eg0KBWxpbWl0GAEgASgFEhIKCnBhZ2VfdG9rZW4YAiABKAkiJwoMUGFnZVJlc3BvbnNlEhcKD25leHRfcGFnZV90b2tlbhgBIAEoCSJkCg1UcmFjZURvY3VtZW50Eg8KB2NvbnRlbnQYASABKAwSNAoGZm9ybWF0GAIgASgOMiQuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VGb3JtYXQSDAoEbmFtZRgDIAEoCSJ+Cg1PdGxwVGVsZW1ldHJ5EjgKBnRyYWNlcxgBIAEoCzIoLm9wZW50ZWxlbWV0cnkucHJvdG8udHJhY2UudjEuVHJhY2VzRGF0YRIzCgRsb2dzGAIgASgLMiUub3BlbnRlbGVtZXRyeS5wcm90by5sb2dzLnYxLkxvZ3NEYXRhIn0KClRyYWNlSW5wdXQSNAoEb3RscBgBIAEoCzImLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk90bHBUZWxlbWV0cnkSOQoJZG9jdW1lbnRzGAIgAygLMiYuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VEb2N1bWVudCqCAQoLVHJhY2VGb3JtYXQSHAoYVFJBQ0VfRk9STUFUX1VOU1BFQ0lGSUVEEAASGgoWVFJBQ0VfRk9STUFUX09UTFBfSlNPThABEhsKF1RSQUNFX0ZPUk1BVF9PVExQX0pTT05MEAISHAoYVFJBQ0VfRk9STUFUX0pBRUdFUl9KU09OEAMqZQoHR3JvdXBCeRIYChRHUk9VUF9CWV9VTlNQRUNJRklFRBAAEhEKDUdST1VQX0JZX0FVVE8QARISCg5HUk9VUF9CWV9UUkFDRRACEhkKFUdST1VQX0JZX0NPTlZFUlNBVElPThADQlFaT2dpdGh1Yi5jb20vYWdlbnRldmFscy1kZXYvYWdlbnRldmFscy9nZW4vZ28vYWdlbnRldmFscy9hcGkvdjFhbHBoYTE7YXBpdjFhbHBoYTFiBnByb3RvMw", [file_opentelemetry_proto_logs_v1_logs, file_opentelemetry_proto_trace_v1_trace]); + +/** + * Describes the message agentevals.api.v1alpha1.PageRequest. + * Use `create(PageRequestSchema)` to create a new message. + */ +export const PageRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_common, 0); + +/** + * Describes the message agentevals.api.v1alpha1.PageResponse. + * Use `create(PageResponseSchema)` to create a new message. + */ +export const PageResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_common, 1); + +/** + * Describes the message agentevals.api.v1alpha1.TraceDocument. + * Use `create(TraceDocumentSchema)` to create a new message. + */ +export const TraceDocumentSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_common, 2); + +/** + * Describes the message agentevals.api.v1alpha1.OtlpTelemetry. + * Use `create(OtlpTelemetrySchema)` to create a new message. + */ +export const OtlpTelemetrySchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_common, 3); + +/** + * Describes the message agentevals.api.v1alpha1.TraceInput. + * Use `create(TraceInputSchema)` to create a new message. + */ +export const TraceInputSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_common, 4); + +/** + * Describes the enum agentevals.api.v1alpha1.TraceFormat. + */ +export const TraceFormatSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_common, 0); + +/** + * @generated from enum agentevals.api.v1alpha1.TraceFormat + */ +export const TraceFormat = /*@__PURE__*/ + tsEnum(TraceFormatSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.GroupBy. + */ +export const GroupBySchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_common, 1); + +/** + * How traces form evaluation units. + * + * @generated from enum agentevals.api.v1alpha1.GroupBy + */ +export const GroupBy = /*@__PURE__*/ + tsEnum(GroupBySchema); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.d.ts new file mode 100644 index 00000000..c8f5348e --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.d.ts @@ -0,0 +1,100 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/conversion.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { JsonObject, Message } from "@bufbuild/protobuf"; +import type { GroupBy, TraceInput } from "./common_pb"; +import type { ConversationView } from "./views_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/conversion.proto. + */ +export declare const file_agentevals_api_v1alpha1_conversion: GenFile; + +/** + * @generated from message agentevals.api.v1alpha1.ConvertTracesRequest + */ +export declare type ConvertTracesRequest = Message<"agentevals.api.v1alpha1.ConvertTracesRequest"> & { + /** + * @generated from field: agentevals.api.v1alpha1.TraceInput traces = 1; + */ + traces?: TraceInput | undefined; + + /** + * Unset means GROUP_BY_TRACE. + * + * @generated from field: agentevals.api.v1alpha1.GroupBy group_by = 2; + */ + groupBy: GroupBy; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ConvertTracesRequest. + * Use `create(ConvertTracesRequestSchema)` to create a new message. + */ +export declare const ConvertTracesRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ConvertedConversation + */ +export declare type ConvertedConversation = Message<"agentevals.api.v1alpha1.ConvertedConversation"> & { + /** + * @generated from field: agentevals.api.v1alpha1.ConversationView conversation = 1; + */ + conversation?: ConversationView | undefined; + + /** + * The conversation as an ADK EvalSet with one case, to use as a golden. Without captured + * content (see TurnView.content_captured) its user turns are empty. + * + * @generated from field: google.protobuf.Struct golden_adk_eval_set = 2; + */ + goldenAdkEvalSet?: JsonObject | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ConvertedConversation. + * Use `create(ConvertedConversationSchema)` to create a new message. + */ +export declare const ConvertedConversationSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ConvertTracesResponse + */ +export declare type ConvertTracesResponse = Message<"agentevals.api.v1alpha1.ConvertTracesResponse"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.ConvertedConversation conversations = 1; + */ + conversations: ConvertedConversation[]; + + /** + * Documents that could not be read. + * + * @generated from field: repeated string warnings = 2; + */ + warnings: string[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ConvertTracesResponse. + * Use `create(ConvertTracesResponseSchema)` to create a new message. + */ +export declare const ConvertTracesResponseSchema: GenMessage; + +/** + * @generated from service agentevals.api.v1alpha1.ConversionService + */ +export declare const ConversionService: GenService<{ + /** + * Reads traces into conversations without evaluating them. + * + * @generated from rpc agentevals.api.v1alpha1.ConversionService.ConvertTraces + */ + convertTraces: { + methodKind: "unary"; + input: typeof ConvertTracesRequestSchema; + output: typeof ConvertTracesResponseSchema; + }, +}>; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.js b/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.js new file mode 100644 index 00000000..22238f0e --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/conversion_pb.js @@ -0,0 +1,42 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/conversion.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_agentevals_api_v1alpha1_common } from "./common_pb"; +import { file_agentevals_api_v1alpha1_views } from "./views_pb"; +import { file_google_protobuf_struct } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/conversion.proto. + */ +export const file_agentevals_api_v1alpha1_conversion = /*@__PURE__*/ + fileDesc("CihhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9jb252ZXJzaW9uLnByb3RvEhdhZ2VudGV2YWxzLmFwaS52MWFscGhhMSJ/ChRDb252ZXJ0VHJhY2VzUmVxdWVzdBIzCgZ0cmFjZXMYASABKAsyIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UcmFjZUlucHV0EjIKCGdyb3VwX2J5GAIgASgOMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR3JvdXBCeSKOAQoVQ29udmVydGVkQ29udmVyc2F0aW9uEj8KDGNvbnZlcnNhdGlvbhgBIAEoCzIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkNvbnZlcnNhdGlvblZpZXcSNAoTZ29sZGVuX2Fka19ldmFsX3NldBgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QicAoVQ29udmVydFRyYWNlc1Jlc3BvbnNlEkUKDWNvbnZlcnNhdGlvbnMYASADKAsyLi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Db252ZXJ0ZWRDb252ZXJzYXRpb24SEAoId2FybmluZ3MYAiADKAkygwEKEUNvbnZlcnNpb25TZXJ2aWNlEm4KDUNvbnZlcnRUcmFjZXMSLS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Db252ZXJ0VHJhY2VzUmVxdWVzdBouLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkNvbnZlcnRUcmFjZXNSZXNwb25zZUJRWk9naXRodWIuY29tL2FnZW50ZXZhbHMtZGV2L2FnZW50ZXZhbHMvZ2VuL2dvL2FnZW50ZXZhbHMvYXBpL3YxYWxwaGExO2FwaXYxYWxwaGExYgZwcm90bzM", [file_agentevals_api_v1alpha1_common, file_agentevals_api_v1alpha1_views, file_google_protobuf_struct]); + +/** + * Describes the message agentevals.api.v1alpha1.ConvertTracesRequest. + * Use `create(ConvertTracesRequestSchema)` to create a new message. + */ +export const ConvertTracesRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_conversion, 0); + +/** + * Describes the message agentevals.api.v1alpha1.ConvertedConversation. + * Use `create(ConvertedConversationSchema)` to create a new message. + */ +export const ConvertedConversationSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_conversion, 1); + +/** + * Describes the message agentevals.api.v1alpha1.ConvertTracesResponse. + * Use `create(ConvertTracesResponseSchema)` to create a new message. + */ +export const ConvertTracesResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_conversion, 2); + +/** + * @generated from service agentevals.api.v1alpha1.ConversionService + */ +export const ConversionService = /*@__PURE__*/ + serviceDesc(file_agentevals_api_v1alpha1_conversion, 0); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts new file mode 100644 index 00000000..b344014e --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts @@ -0,0 +1,784 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/evaluation.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { JsonObject, Message } from "@bufbuild/protobuf"; +import type { GroupBy, TraceInput } from "./common_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/evaluation.proto. + */ +export declare const file_agentevals_api_v1alpha1_evaluation: GenFile; + +/** + * @generated from message agentevals.api.v1alpha1.ListMetricsRequest + */ +export declare type ListMetricsRequest = Message<"agentevals.api.v1alpha1.ListMetricsRequest"> & { +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListMetricsRequest. + * Use `create(ListMetricsRequestSchema)` to create a new message. + */ +export declare const ListMetricsRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.Metric + */ +export declare type Metric = Message<"agentevals.api.v1alpha1.Metric"> & { + /** + * @generated from field: string name = 1; + */ + name: string; + + /** + * @generated from field: string category = 2; + */ + category: string; + + /** + * @generated from field: string description = 3; + */ + description: string; + + /** + * Needs a golden eval set. + * + * @generated from field: bool requires_eval_set = 4; + */ + requiresEvalSet: boolean; + + /** + * Calls a judge model. + * + * @generated from field: bool requires_llm = 5; + */ + requiresLlm: boolean; + + /** + * Needs Google Cloud credentials. + * + * @generated from field: bool requires_gcp = 6; + */ + requiresGcp: boolean; + + /** + * Needs rubrics in the eval set. + * + * @generated from field: bool requires_rubrics = 7; + */ + requiresRubrics: boolean; + + /** + * Usable with recorded traces. + * + * @generated from field: bool working = 8; + */ + working: boolean; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Metric. + * Use `create(MetricSchema)` to create a new message. + */ +export declare const MetricSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListMetricsResponse + */ +export declare type ListMetricsResponse = Message<"agentevals.api.v1alpha1.ListMetricsResponse"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.Metric metrics = 1; + */ + metrics: Metric[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListMetricsResponse. + * Use `create(ListMetricsResponseSchema)` to create a new message. + */ +export declare const ListMetricsResponseSchema: GenMessage; + +/** + * A built-in metric, by name (see ListMetrics), with optional overrides. + * + * @generated from message agentevals.api.v1alpha1.BuiltinEvaluator + */ +export declare type BuiltinEvaluator = Message<"agentevals.api.v1alpha1.BuiltinEvaluator"> & { + /** + * @generated from field: string name = 1; + */ + name: string; + + /** + * 0 to 1. Unset uses the metric's default. + * + * @generated from field: optional double threshold = 2; + */ + threshold?: number | undefined; + + /** + * @generated from field: string judge_model = 3; + */ + judgeModel: string; + + /** + * @generated from field: agentevals.api.v1alpha1.TrajectoryMatchType trajectory_match_type = 4; + */ + trajectoryMatchType: TrajectoryMatchType; + + /** + * Name of a credential_refs entry whose value is the judge API key. + * + * @generated from field: string credential_ref = 5; + */ + credentialRef: string; + + /** + * Base URL of an OpenAI compatible judge endpoint. + * + * @generated from field: string judge_base_url = 6; + */ + judgeBaseUrl: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.BuiltinEvaluator. + * Use `create(BuiltinEvaluatorSchema)` to create a new message. + */ +export declare const BuiltinEvaluatorSchema: GenMessage; + +/** + * An evaluator file on the server, run as a subprocess. + * + * @generated from message agentevals.api.v1alpha1.CodeEvaluator + */ +export declare type CodeEvaluator = Message<"agentevals.api.v1alpha1.CodeEvaluator"> & { + /** + * @generated from field: string name = 1; + */ + name: string; + + /** + * 0 to 1. Unset means 0.5. + * + * @generated from field: optional double threshold = 2; + */ + threshold?: number | undefined; + + /** + * Seconds. Unset means 30. + * + * @generated from field: optional int32 timeout = 3; + */ + timeout?: number | undefined; + + /** + * @generated from field: google.protobuf.Struct config = 4; + */ + config?: JsonObject | undefined; + + /** + * Unset means local. + * + * @generated from field: string executor = 5; + */ + executor: string; + + /** + * Path of the evaluator file (.py, .js, .ts and so on) on the server. + * + * @generated from field: string path = 6; + */ + path: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.CodeEvaluator. + * Use `create(CodeEvaluatorSchema)` to create a new message. + */ +export declare const CodeEvaluatorSchema: GenMessage; + +/** + * An evaluator fetched from a remote source. + * + * @generated from message agentevals.api.v1alpha1.RemoteEvaluator + */ +export declare type RemoteEvaluator = Message<"agentevals.api.v1alpha1.RemoteEvaluator"> & { + /** + * @generated from field: string name = 1; + */ + name: string; + + /** + * 0 to 1. Unset means 0.5. + * + * @generated from field: optional double threshold = 2; + */ + threshold?: number | undefined; + + /** + * Seconds. Unset means 30. + * + * @generated from field: optional int32 timeout = 3; + */ + timeout?: number | undefined; + + /** + * @generated from field: google.protobuf.Struct config = 4; + */ + config?: JsonObject | undefined; + + /** + * Unset means local. + * + * @generated from field: string executor = 5; + */ + executor: string; + + /** + * Unset means github. + * + * @generated from field: string source = 6; + */ + source: string; + + /** + * A relative path within the source. + * + * @generated from field: string ref = 7; + */ + ref: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.RemoteEvaluator. + * Use `create(RemoteEvaluatorSchema)` to create a new message. + */ +export declare const RemoteEvaluatorSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.Evaluator + */ +export declare type Evaluator = Message<"agentevals.api.v1alpha1.Evaluator"> & { + /** + * @generated from oneof agentevals.api.v1alpha1.Evaluator.kind + */ + kind: { + /** + * @generated from field: agentevals.api.v1alpha1.BuiltinEvaluator builtin = 1; + */ + value: BuiltinEvaluator; + case: "builtin"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.CodeEvaluator code = 2; + */ + value: CodeEvaluator; + case: "code"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.RemoteEvaluator remote = 3; + */ + value: RemoteEvaluator; + case: "remote"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Evaluator. + * Use `create(EvaluatorSchema)` to create a new message. + */ +export declare const EvaluatorSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.EvalParams + */ +export declare type EvalParams = Message<"agentevals.api.v1alpha1.EvalParams"> & { + /** + * Names must be unique. Empty means tool_trajectory_avg_score. + * + * @generated from field: repeated agentevals.api.v1alpha1.Evaluator evaluators = 1; + */ + evaluators: Evaluator[]; + + /** + * At least 1. Unset means 10. + * + * @generated from field: optional int32 max_concurrent_traces = 2; + */ + maxConcurrentTraces?: number | undefined; + + /** + * At least 1. Unset means 5. + * + * @generated from field: optional int32 max_concurrent_evals = 3; + */ + maxConcurrentEvals?: number | undefined; + + /** + * Unset means GROUP_BY_AUTO. + * + * @generated from field: agentevals.api.v1alpha1.GroupBy group_by = 4; + */ + groupBy: GroupBy; +}; + +/** + * Describes the message agentevals.api.v1alpha1.EvalParams. + * Use `create(EvalParamsSchema)` to create a new message. + */ +export declare const EvalParamsSchema: GenMessage; + +/** + * A metric's result for one turn. + * + * @generated from message agentevals.api.v1alpha1.InvocationResult + */ +export declare type InvocationResult = Message<"agentevals.api.v1alpha1.InvocationResult"> & { + /** + * Unset when the turn has no score. + * + * @generated from field: optional double score = 1; + */ + score?: number | undefined; + + /** + * @generated from field: agentevals.api.v1alpha1.EvalStatus status = 2; + */ + status: EvalStatus; +}; + +/** + * Describes the message agentevals.api.v1alpha1.InvocationResult. + * Use `create(InvocationResultSchema)` to create a new message. + */ +export declare const InvocationResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.MetricResult + */ +export declare type MetricResult = Message<"agentevals.api.v1alpha1.MetricResult"> & { + /** + * @generated from field: string metric_name = 1; + */ + metricName: string; + + /** + * @generated from field: optional double score = 2; + */ + score?: number | undefined; + + /** + * @generated from field: agentevals.api.v1alpha1.EvalStatus eval_status = 3; + */ + evalStatus: EvalStatus; + + /** + * One entry per turn, in turn order. + * + * @generated from field: repeated agentevals.api.v1alpha1.InvocationResult per_invocation = 4; + */ + perInvocation: InvocationResult[]; + + /** + * @generated from field: optional string error = 5; + */ + error?: string | undefined; + + /** + * error.type. + * + * @generated from field: optional string error_type = 6; + */ + errorType?: string | undefined; + + /** + * Metric specific details. + * + * @generated from field: google.protobuf.Struct details = 7; + */ + details?: JsonObject | undefined; + + /** + * @generated from field: optional double duration_ms = 8; + */ + durationMs?: number | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.MetricResult. + * Use `create(MetricResultSchema)` to create a new message. + */ +export declare const MetricResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.TurnRef + */ +export declare type TurnRef = Message<"agentevals.api.v1alpha1.TurnRef"> & { + /** + * @generated from field: string trace_id = 1; + */ + traceId: string; + + /** + * @generated from field: string span_id = 2; + */ + spanId: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TurnRef. + * Use `create(TurnRefSchema)` to create a new message. + */ +export declare const TurnRefSchema: GenMessage; + +/** + * The results of one evaluation group: a trace, or a conversation of several traces. + * + * @generated from message agentevals.api.v1alpha1.TraceResult + */ +export declare type TraceResult = Message<"agentevals.api.v1alpha1.TraceResult"> & { + /** + * The group's earliest trace. + * + * @generated from field: string trace_id = 1; + */ + traceId: string; + + /** + * @generated from field: int32 num_invocations = 2; + */ + numInvocations: number; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.MetricResult metric_results = 3; + */ + metricResults: MetricResult[]; + + /** + * @generated from field: repeated string conversion_warnings = 4; + */ + conversionWarnings: string[]; + + /** + * @generated from field: google.protobuf.Struct performance_metrics = 5; + */ + performanceMetrics?: JsonObject | undefined; + + /** + * service.name. + * + * @generated from field: string service_name = 6; + */ + serviceName: string; + + /** + * gen_ai.agent.name. + * + * @generated from field: string agent_name = 7; + */ + agentName: string; + + /** + * @generated from field: repeated string trace_ids = 8; + */ + traceIds: string[]; + + /** + * gen_ai.conversation.id. + * + * @generated from field: string conversation_id = 9; + */ + conversationId: string; + + /** + * @generated from field: string group_key = 10; + */ + groupKey: string; + + /** + * The eval set case the group was compared with. + * + * @generated from field: string eval_case_id = 11; + */ + evalCaseId: string; + + /** + * How the case was chosen. + * + * @generated from field: string eval_case_match = 12; + */ + evalCaseMatch: string; + + /** + * The judged turns, in order. + * + * @generated from field: repeated agentevals.api.v1alpha1.TurnRef turn_refs = 13; + */ + turnRefs: TurnRef[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TraceResult. + * Use `create(TraceResultSchema)` to create a new message. + */ +export declare const TraceResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.RunResult + */ +export declare type RunResult = Message<"agentevals.api.v1alpha1.RunResult"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.TraceResult trace_results = 1; + */ + traceResults: TraceResult[]; + + /** + * @generated from field: repeated string errors = 2; + */ + errors: string[]; + + /** + * @generated from field: google.protobuf.Struct performance_metrics = 3; + */ + performanceMetrics?: JsonObject | undefined; + + /** + * Set when the evaluation was recorded as a run. + * + * @generated from field: string run_id = 4; + */ + runId: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.RunResult. + * Use `create(RunResultSchema)` to create a new message. + */ +export declare const RunResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.EvaluateTracesRequest + */ +export declare type EvaluateTracesRequest = Message<"agentevals.api.v1alpha1.EvaluateTracesRequest"> & { + /** + * @generated from field: agentevals.api.v1alpha1.TraceInput traces = 1; + */ + traces?: TraceInput | undefined; + + /** + * The golden. Required by metrics with requires_eval_set. + * + * @generated from oneof agentevals.api.v1alpha1.EvaluateTracesRequest.eval_set + */ + evalSet: { + /** + * An ADK EvalSet as JSON. + * + * @generated from field: google.protobuf.Struct adk_eval_set = 2; + */ + value: JsonObject; + case: "adkEvalSet"; + } | { case: undefined; value?: undefined }; + + /** + * @generated from field: agentevals.api.v1alpha1.EvalParams params = 3; + */ + params?: EvalParams | undefined; + + /** + * Logical credential name to a secret reference, for example + * {"kind": "kubernetes", "namespace": "...", "name": "...", "key": "..."} or + * {"kind": "env", "name": "..."}. Values are resolved per request and never returned. + * + * @generated from field: map credential_refs = 4; + */ + credentialRefs: { [key: string]: JsonObject }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.EvaluateTracesRequest. + * Use `create(EvaluateTracesRequestSchema)` to create a new message. + */ +export declare const EvaluateTracesRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.EvaluationProgress + */ +export declare type EvaluationProgress = Message<"agentevals.api.v1alpha1.EvaluationProgress"> & { + /** + * @generated from field: string message = 1; + */ + message: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.EvaluationProgress. + * Use `create(EvaluationProgressSchema)` to create a new message. + */ +export declare const EvaluationProgressSchema: GenMessage; + +/** + * A group's performance metrics, sent before it is evaluated. + * + * @generated from message agentevals.api.v1alpha1.GroupPerformance + */ +export declare type GroupPerformance = Message<"agentevals.api.v1alpha1.GroupPerformance"> & { + /** + * The group's earliest trace. + * + * @generated from field: string trace_id = 1; + */ + traceId: string; + + /** + * @generated from field: google.protobuf.Struct performance_metrics = 2; + */ + performanceMetrics?: JsonObject | undefined; + + /** + * @generated from field: google.protobuf.Struct trace_metadata = 3; + */ + traceMetadata?: JsonObject | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GroupPerformance. + * Use `create(GroupPerformanceSchema)` to create a new message. + */ +export declare const GroupPerformanceSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.EvaluateTracesResponse + */ +export declare type EvaluateTracesResponse = Message<"agentevals.api.v1alpha1.EvaluateTracesResponse"> & { + /** + * @generated from oneof agentevals.api.v1alpha1.EvaluateTracesResponse.event + */ + event: { + /** + * @generated from field: agentevals.api.v1alpha1.EvaluationProgress progress = 1; + */ + value: EvaluationProgress; + case: "progress"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.GroupPerformance performance = 2; + */ + value: GroupPerformance; + case: "performance"; + } | { + /** + * A group's results so far. + * + * @generated from field: agentevals.api.v1alpha1.TraceResult trace_result = 3; + */ + value: TraceResult; + case: "traceResult"; + } | { + /** + * The final result. + * + * @generated from field: agentevals.api.v1alpha1.RunResult done = 4; + */ + value: RunResult; + case: "done"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.EvaluateTracesResponse. + * Use `create(EvaluateTracesResponseSchema)` to create a new message. + */ +export declare const EvaluateTracesResponseSchema: GenMessage; + +/** + * @generated from enum agentevals.api.v1alpha1.TrajectoryMatchType + */ +export enum TrajectoryMatchType { + /** + * The metric's default. + * + * @generated from enum value: TRAJECTORY_MATCH_TYPE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: TRAJECTORY_MATCH_TYPE_EXACT = 1; + */ + EXACT = 1, + + /** + * @generated from enum value: TRAJECTORY_MATCH_TYPE_IN_ORDER = 2; + */ + IN_ORDER = 2, + + /** + * @generated from enum value: TRAJECTORY_MATCH_TYPE_ANY_ORDER = 3; + */ + ANY_ORDER = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.TrajectoryMatchType. + */ +export declare const TrajectoryMatchTypeSchema: GenEnum; + +/** + * @generated from enum agentevals.api.v1alpha1.EvalStatus + */ +export enum EvalStatus { + /** + * @generated from enum value: EVAL_STATUS_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: EVAL_STATUS_PASSED = 1; + */ + PASSED = 1, + + /** + * @generated from enum value: EVAL_STATUS_FAILED = 2; + */ + FAILED = 2, + + /** + * @generated from enum value: EVAL_STATUS_NOT_EVALUATED = 3; + */ + NOT_EVALUATED = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.EvalStatus. + */ +export declare const EvalStatusSchema: GenEnum; + +/** + * @generated from service agentevals.api.v1alpha1.EvaluationService + */ +export declare const EvaluationService: GenService<{ + /** + * @generated from rpc agentevals.api.v1alpha1.EvaluationService.ListMetrics + */ + listMetrics: { + methodKind: "unary"; + input: typeof ListMetricsRequestSchema; + output: typeof ListMetricsResponseSchema; + }, + /** + * Evaluates traces. Streams progress, each group's performance metrics before evaluation, each + * group's results, and the final result, after which the stream ends. When run storage is + * configured, the evaluation is also recorded as a run. + * + * @generated from rpc agentevals.api.v1alpha1.EvaluationService.EvaluateTraces + */ + evaluateTraces: { + methodKind: "server_streaming"; + input: typeof EvaluateTracesRequestSchema; + output: typeof EvaluateTracesResponseSchema; + }, +}>; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js new file mode 100644 index 00000000..f3f758b7 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js @@ -0,0 +1,163 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/evaluation.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, serviceDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_agentevals_api_v1alpha1_common } from "./common_pb"; +import { file_google_protobuf_struct } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/evaluation.proto. + */ +export const file_agentevals_api_v1alpha1_evaluation = /*@__PURE__*/ + fileDesc("CihhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9ldmFsdWF0aW9uLnByb3RvEhdhZ2VudGV2YWxzLmFwaS52MWFscGhhMSIUChJMaXN0TWV0cmljc1JlcXVlc3QirwEKBk1ldHJpYxIMCgRuYW1lGAEgASgJEhAKCGNhdGVnb3J5GAIgASgJEhMKC2Rlc2NyaXB0aW9uGAMgASgJEhkKEXJlcXVpcmVzX2V2YWxfc2V0GAQgASgIEhQKDHJlcXVpcmVzX2xsbRgFIAEoCBIUCgxyZXF1aXJlc19nY3AYBiABKAgSGAoQcmVxdWlyZXNfcnVicmljcxgHIAEoCBIPCgd3b3JraW5nGAggASgIIkcKE0xpc3RNZXRyaWNzUmVzcG9uc2USMAoHbWV0cmljcxgBIAMoCzIfLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk1ldHJpYyLYAQoQQnVpbHRpbkV2YWx1YXRvchIMCgRuYW1lGAEgASgJEhYKCXRocmVzaG9sZBgCIAEoAUgAiAEBEhMKC2p1ZGdlX21vZGVsGAMgASgJEksKFXRyYWplY3RvcnlfbWF0Y2hfdHlwZRgEIAEoDjIsLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWplY3RvcnlNYXRjaFR5cGUSFgoOY3JlZGVudGlhbF9yZWYYBSABKAkSFgoOanVkZ2VfYmFzZV91cmwYBiABKAlCDAoKX3RocmVzaG9sZCKuAQoNQ29kZUV2YWx1YXRvchIMCgRuYW1lGAEgASgJEhYKCXRocmVzaG9sZBgCIAEoAUgAiAEBEhQKB3RpbWVvdXQYAyABKAVIAYgBARInCgZjb25maWcYBCABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhAKCGV4ZWN1dG9yGAUgASgJEgwKBHBhdGgYBiABKAlCDAoKX3RocmVzaG9sZEIKCghfdGltZW91dCK/AQoPUmVtb3RlRXZhbHVhdG9yEgwKBG5hbWUYASABKAkSFgoJdGhyZXNob2xkGAIgASgBSACIAQESFAoHdGltZW91dBgDIAEoBUgBiAEBEicKBmNvbmZpZxgEIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSEAoIZXhlY3V0b3IYBSABKAkSDgoGc291cmNlGAYgASgJEgsKA3JlZhgHIAEoCUIMCgpfdGhyZXNob2xkQgoKCF90aW1lb3V0IsUBCglFdmFsdWF0b3ISPAoHYnVpbHRpbhgBIAEoCzIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkJ1aWx0aW5FdmFsdWF0b3JIABI2CgRjb2RlGAIgASgLMiYuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuQ29kZUV2YWx1YXRvckgAEjoKBnJlbW90ZRgDIAEoCzIoLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJlbW90ZUV2YWx1YXRvckgAQgYKBGtpbmQi8gEKCkV2YWxQYXJhbXMSNgoKZXZhbHVhdG9ycxgBIAMoCzIiLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRvchIiChVtYXhfY29uY3VycmVudF90cmFjZXMYAiABKAVIAIgBARIhChRtYXhfY29uY3VycmVudF9ldmFscxgDIAEoBUgBiAEBEjIKCGdyb3VwX2J5GAQgASgOMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR3JvdXBCeUIYChZfbWF4X2NvbmN1cnJlbnRfdHJhY2VzQhcKFV9tYXhfY29uY3VycmVudF9ldmFscyJlChBJbnZvY2F0aW9uUmVzdWx0EhIKBXNjb3JlGAEgASgBSACIAQESMwoGc3RhdHVzGAIgASgOMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFN0YXR1c0IICgZfc2NvcmUi2AIKDE1ldHJpY1Jlc3VsdBITCgttZXRyaWNfbmFtZRgBIAEoCRISCgVzY29yZRgCIAEoAUgAiAEBEjgKC2V2YWxfc3RhdHVzGAMgASgOMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFN0YXR1cxJBCg5wZXJfaW52b2NhdGlvbhgEIAMoCzIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkludm9jYXRpb25SZXN1bHQSEgoFZXJyb3IYBSABKAlIAYgBARIXCgplcnJvcl90eXBlGAYgASgJSAKIAQESKAoHZGV0YWlscxgHIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSGAoLZHVyYXRpb25fbXMYCCABKAFIA4gBAUIICgZfc2NvcmVCCAoGX2Vycm9yQg0KC19lcnJvcl90eXBlQg4KDF9kdXJhdGlvbl9tcyIsCgdUdXJuUmVmEhAKCHRyYWNlX2lkGAEgASgJEg8KB3NwYW5faWQYAiABKAkilwMKC1RyYWNlUmVzdWx0EhAKCHRyYWNlX2lkGAEgASgJEhcKD251bV9pbnZvY2F0aW9ucxgCIAEoBRI9Cg5tZXRyaWNfcmVzdWx0cxgDIAMoCzIlLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk1ldHJpY1Jlc3VsdBIbChNjb252ZXJzaW9uX3dhcm5pbmdzGAQgAygJEjQKE3BlcmZvcm1hbmNlX21ldHJpY3MYBSABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhQKDHNlcnZpY2VfbmFtZRgGIAEoCRISCgphZ2VudF9uYW1lGAcgASgJEhEKCXRyYWNlX2lkcxgIIAMoCRIXCg9jb252ZXJzYXRpb25faWQYCSABKAkSEQoJZ3JvdXBfa2V5GAogASgJEhQKDGV2YWxfY2FzZV9pZBgLIAEoCRIXCg9ldmFsX2Nhc2VfbWF0Y2gYDCABKAkSMwoJdHVybl9yZWZzGA0gAygLMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHVyblJlZiKeAQoJUnVuUmVzdWx0EjsKDXRyYWNlX3Jlc3VsdHMYASADKAsyJC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UcmFjZVJlc3VsdBIOCgZlcnJvcnMYAiADKAkSNAoTcGVyZm9ybWFuY2VfbWV0cmljcxgDIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSDgoGcnVuX2lkGAQgASgJIusCChVFdmFsdWF0ZVRyYWNlc1JlcXVlc3QSMwoGdHJhY2VzGAEgASgLMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VJbnB1dBIvCgxhZGtfZXZhbF9zZXQYAiABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0SAASMwoGcGFyYW1zGAMgASgLMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFBhcmFtcxJbCg9jcmVkZW50aWFsX3JlZnMYBCADKAsyQi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1JlcXVlc3QuQ3JlZGVudGlhbFJlZnNFbnRyeRpOChNDcmVkZW50aWFsUmVmc0VudHJ5EgsKA2tleRgBIAEoCRImCgV2YWx1ZRgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3Q6AjgBQgoKCGV2YWxfc2V0IiUKEkV2YWx1YXRpb25Qcm9ncmVzcxIPCgdtZXNzYWdlGAEgASgJIosBChBHcm91cFBlcmZvcm1hbmNlEhAKCHRyYWNlX2lkGAEgASgJEjQKE3BlcmZvcm1hbmNlX21ldHJpY3MYAiABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0Ei8KDnRyYWNlX21ldGFkYXRhGAMgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdCKWAgoWRXZhbHVhdGVUcmFjZXNSZXNwb25zZRI/Cghwcm9ncmVzcxgBIAEoCzIrLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRpb25Qcm9ncmVzc0gAEkAKC3BlcmZvcm1hbmNlGAIgASgLMikuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR3JvdXBQZXJmb3JtYW5jZUgAEjwKDHRyYWNlX3Jlc3VsdBgDIAEoCzIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWNlUmVzdWx0SAASMgoEZG9uZRgEIAEoCzIiLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1blJlc3VsdEgAQgcKBWV2ZW50KqYBChNUcmFqZWN0b3J5TWF0Y2hUeXBlEiUKIVRSQUpFQ1RPUllfTUFUQ0hfVFlQRV9VTlNQRUNJRklFRBAAEh8KG1RSQUpFQ1RPUllfTUFUQ0hfVFlQRV9FWEFDVBABEiIKHlRSQUpFQ1RPUllfTUFUQ0hfVFlQRV9JTl9PUkRFUhACEiMKH1RSQUpFQ1RPUllfTUFUQ0hfVFlQRV9BTllfT1JERVIQAyp4CgpFdmFsU3RhdHVzEhsKF0VWQUxfU1RBVFVTX1VOU1BFQ0lGSUVEEAASFgoSRVZBTF9TVEFUVVNfUEFTU0VEEAESFgoSRVZBTF9TVEFUVVNfRkFJTEVEEAISHQoZRVZBTF9TVEFUVVNfTk9UX0VWQUxVQVRFRBADMvIBChFFdmFsdWF0aW9uU2VydmljZRJoCgtMaXN0TWV0cmljcxIrLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RNZXRyaWNzUmVxdWVzdBosLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RNZXRyaWNzUmVzcG9uc2UScwoORXZhbHVhdGVUcmFjZXMSLi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1JlcXVlc3QaLy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1Jlc3BvbnNlMAFCUVpPZ2l0aHViLmNvbS9hZ2VudGV2YWxzLWRldi9hZ2VudGV2YWxzL2dlbi9nby9hZ2VudGV2YWxzL2FwaS92MWFscGhhMTthcGl2MWFscGhhMWIGcHJvdG8z", [file_agentevals_api_v1alpha1_common, file_google_protobuf_struct]); + +/** + * Describes the message agentevals.api.v1alpha1.ListMetricsRequest. + * Use `create(ListMetricsRequestSchema)` to create a new message. + */ +export const ListMetricsRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 0); + +/** + * Describes the message agentevals.api.v1alpha1.Metric. + * Use `create(MetricSchema)` to create a new message. + */ +export const MetricSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 1); + +/** + * Describes the message agentevals.api.v1alpha1.ListMetricsResponse. + * Use `create(ListMetricsResponseSchema)` to create a new message. + */ +export const ListMetricsResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 2); + +/** + * Describes the message agentevals.api.v1alpha1.BuiltinEvaluator. + * Use `create(BuiltinEvaluatorSchema)` to create a new message. + */ +export const BuiltinEvaluatorSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 3); + +/** + * Describes the message agentevals.api.v1alpha1.CodeEvaluator. + * Use `create(CodeEvaluatorSchema)` to create a new message. + */ +export const CodeEvaluatorSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 4); + +/** + * Describes the message agentevals.api.v1alpha1.RemoteEvaluator. + * Use `create(RemoteEvaluatorSchema)` to create a new message. + */ +export const RemoteEvaluatorSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 5); + +/** + * Describes the message agentevals.api.v1alpha1.Evaluator. + * Use `create(EvaluatorSchema)` to create a new message. + */ +export const EvaluatorSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 6); + +/** + * Describes the message agentevals.api.v1alpha1.EvalParams. + * Use `create(EvalParamsSchema)` to create a new message. + */ +export const EvalParamsSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 7); + +/** + * Describes the message agentevals.api.v1alpha1.InvocationResult. + * Use `create(InvocationResultSchema)` to create a new message. + */ +export const InvocationResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 8); + +/** + * Describes the message agentevals.api.v1alpha1.MetricResult. + * Use `create(MetricResultSchema)` to create a new message. + */ +export const MetricResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 9); + +/** + * Describes the message agentevals.api.v1alpha1.TurnRef. + * Use `create(TurnRefSchema)` to create a new message. + */ +export const TurnRefSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 10); + +/** + * Describes the message agentevals.api.v1alpha1.TraceResult. + * Use `create(TraceResultSchema)` to create a new message. + */ +export const TraceResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 11); + +/** + * Describes the message agentevals.api.v1alpha1.RunResult. + * Use `create(RunResultSchema)` to create a new message. + */ +export const RunResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 12); + +/** + * Describes the message agentevals.api.v1alpha1.EvaluateTracesRequest. + * Use `create(EvaluateTracesRequestSchema)` to create a new message. + */ +export const EvaluateTracesRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 13); + +/** + * Describes the message agentevals.api.v1alpha1.EvaluationProgress. + * Use `create(EvaluationProgressSchema)` to create a new message. + */ +export const EvaluationProgressSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 14); + +/** + * Describes the message agentevals.api.v1alpha1.GroupPerformance. + * Use `create(GroupPerformanceSchema)` to create a new message. + */ +export const GroupPerformanceSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 15); + +/** + * Describes the message agentevals.api.v1alpha1.EvaluateTracesResponse. + * Use `create(EvaluateTracesResponseSchema)` to create a new message. + */ +export const EvaluateTracesResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 16); + +/** + * Describes the enum agentevals.api.v1alpha1.TrajectoryMatchType. + */ +export const TrajectoryMatchTypeSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_evaluation, 0); + +/** + * @generated from enum agentevals.api.v1alpha1.TrajectoryMatchType + */ +export const TrajectoryMatchType = /*@__PURE__*/ + tsEnum(TrajectoryMatchTypeSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.EvalStatus. + */ +export const EvalStatusSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_evaluation, 1); + +/** + * @generated from enum agentevals.api.v1alpha1.EvalStatus + */ +export const EvalStatus = /*@__PURE__*/ + tsEnum(EvalStatusSchema); + +/** + * @generated from service agentevals.api.v1alpha1.EvaluationService + */ +export const EvaluationService = /*@__PURE__*/ + serviceDesc(file_agentevals_api_v1alpha1_evaluation, 0); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/runs_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/runs_pb.d.ts new file mode 100644 index 00000000..37ee8913 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/runs_pb.d.ts @@ -0,0 +1,785 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/runs.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { JsonObject, Message } from "@bufbuild/protobuf"; +import type { OtlpTelemetry, PageRequest, PageResponse, TraceDocument, TraceFormat } from "./common_pb"; +import type { EvalParams, InvocationResult } from "./evaluation_pb"; +import type { Timestamp } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/runs.proto. + */ +export declare const file_agentevals_api_v1alpha1_runs: GenFile; + +/** + * A trace fetched with GET base_url/trace_id and the run's context headers. + * + * @generated from message agentevals.api.v1alpha1.HttpTraceTarget + */ +export declare type HttpTraceTarget = Message<"agentevals.api.v1alpha1.HttpTraceTarget"> & { + /** + * @generated from field: string base_url = 1; + */ + baseUrl: string; + + /** + * 16 or 32 hex characters. + * + * @generated from field: string trace_id = 2; + */ + traceId: string; + + /** + * TRACE_FORMAT_OTLP_JSON or TRACE_FORMAT_JAEGER_JSON. + * + * @generated from field: agentevals.api.v1alpha1.TraceFormat format = 3; + */ + format: TraceFormat; +}; + +/** + * Describes the message agentevals.api.v1alpha1.HttpTraceTarget. + * Use `create(HttpTraceTargetSchema)` to create a new message. + */ +export declare const HttpTraceTargetSchema: GenMessage; + +/** + * A run recorded from a synchronous upload. The traces are not kept, so it cannot run again. + * + * @generated from message agentevals.api.v1alpha1.UploadedTraces + */ +export declare type UploadedTraces = Message<"agentevals.api.v1alpha1.UploadedTraces"> & { + /** + * @generated from field: int32 trace_count = 1; + */ + traceCount: number; + + /** + * @generated from field: repeated string trace_files = 2; + */ + traceFiles: string[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.UploadedTraces. + * Use `create(UploadedTracesSchema)` to create a new message. + */ +export declare const UploadedTracesSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.TraceTarget + */ +export declare type TraceTarget = Message<"agentevals.api.v1alpha1.TraceTarget"> & { + /** + * @generated from oneof agentevals.api.v1alpha1.TraceTarget.kind + */ + kind: { + /** + * A trace document in the spec, OTLP/JSON or Jaeger JSON. + * + * @generated from field: agentevals.api.v1alpha1.TraceDocument inline = 1; + */ + value: TraceDocument; + case: "inline"; + } | { + /** + * OTLP protobuf. Stored, and returned, as an inline OTLP/JSON document. + * + * @generated from field: agentevals.api.v1alpha1.OtlpTelemetry otlp = 2; + */ + value: OtlpTelemetry; + case: "otlp"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.HttpTraceTarget http = 3; + */ + value: HttpTraceTarget; + case: "http"; + } | { + /** + * Output only. + * + * @generated from field: agentevals.api.v1alpha1.UploadedTraces uploaded = 4; + */ + value: UploadedTraces; + case: "uploaded"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TraceTarget. + * Use `create(TraceTargetSchema)` to create a new message. + */ +export declare const TraceTargetSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.RunContext + */ +export declare type RunContext = Message<"agentevals.api.v1alpha1.RunContext"> & { + /** + * Headers for fetching an http target. Values are returned redacted. + * + * @generated from field: map headers = 1; + */ + headers: { [key: string]: string }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.RunContext. + * Use `create(RunContextSchema)` to create a new message. + */ +export declare const RunContextSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.RunSpec + */ +export declare type RunSpec = Message<"agentevals.api.v1alpha1.RunSpec"> & { + /** + * @generated from field: agentevals.api.v1alpha1.RunApproach approach = 1; + */ + approach: RunApproach; + + /** + * @generated from field: agentevals.api.v1alpha1.TraceTarget target = 2; + */ + target?: TraceTarget | undefined; + + /** + * The golden. + * + * @generated from oneof agentevals.api.v1alpha1.RunSpec.eval_set + */ + evalSet: { + /** + * An ADK EvalSet as JSON. + * + * @generated from field: google.protobuf.Struct adk_eval_set = 3; + */ + value: JsonObject; + case: "adkEvalSet"; + } | { case: undefined; value?: undefined }; + + /** + * @generated from field: agentevals.api.v1alpha1.EvalParams eval_config = 4; + */ + evalConfig?: EvalParams | undefined; + + /** + * Result sinks, for example {"kind": "http_webhook", "url": "...", "headers": {...}}. Header + * values are returned redacted. + * + * @generated from field: repeated google.protobuf.Struct sinks = 5; + */ + sinks: JsonObject[]; + + /** + * @generated from field: agentevals.api.v1alpha1.RunContext context = 6; + */ + context?: RunContext | undefined; + + /** + * Logical credential name to a secret reference, as in EvaluateTracesRequest. + * + * @generated from field: map credential_refs = 7; + */ + credentialRefs: { [key: string]: JsonObject }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.RunSpec. + * Use `create(RunSpecSchema)` to create a new message. + */ +export declare const RunSpecSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.Run + */ +export declare type Run = Message<"agentevals.api.v1alpha1.Run"> & { + /** + * A UUID. + * + * @generated from field: string run_id = 1; + */ + runId: string; + + /** + * @generated from field: agentevals.api.v1alpha1.RunStatus status = 2; + */ + status: RunStatus; + + /** + * @generated from field: agentevals.api.v1alpha1.RunSpec spec = 3; + */ + spec?: RunSpec | undefined; + + /** + * @generated from field: int32 attempt = 4; + */ + attempt: number; + + /** + * @generated from field: string worker_id = 5; + */ + workerId: string; + + /** + * @generated from field: string error = 6; + */ + error: string; + + /** + * @generated from field: google.protobuf.Struct summary = 7; + */ + summary?: JsonObject | undefined; + + /** + * @generated from field: google.protobuf.Timestamp created_at = 8; + */ + createdAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Timestamp started_at = 9; + */ + startedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Timestamp finished_at = 10; + */ + finishedAt?: Timestamp | undefined; + + /** + * @generated from field: bool cancel_requested = 11; + */ + cancelRequested: boolean; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Run. + * Use `create(RunSchema)` to create a new message. + */ +export declare const RunSchema: GenMessage; + +/** + * One evaluator's result for one eval set item. + * + * @generated from message agentevals.api.v1alpha1.Result + */ +export declare type Result = Message<"agentevals.api.v1alpha1.Result"> & { + /** + * Stable: a SHA-256 of run id, item id and evaluator name. + * + * @generated from field: string result_id = 1; + */ + resultId: string; + + /** + * @generated from field: string run_id = 2; + */ + runId: string; + + /** + * @generated from field: string eval_set_item_id = 3; + */ + evalSetItemId: string; + + /** + * @generated from field: string eval_set_item_name = 4; + */ + evalSetItemName: string; + + /** + * @generated from field: string evaluator_name = 5; + */ + evaluatorName: string; + + /** + * @generated from field: agentevals.api.v1alpha1.EvaluatorType evaluator_type = 6; + */ + evaluatorType: EvaluatorType; + + /** + * @generated from field: agentevals.api.v1alpha1.ResultStatus status = 7; + */ + status: ResultStatus; + + /** + * @generated from field: optional double score = 8; + */ + score?: number | undefined; + + /** + * One entry per turn, in turn order; status is not recorded. + * + * @generated from field: repeated agentevals.api.v1alpha1.InvocationResult per_invocation = 9; + */ + perInvocation: InvocationResult[]; + + /** + * @generated from field: string trace_id = 10; + */ + traceId: string; + + /** + * @generated from field: string span_id = 11; + */ + spanId: string; + + /** + * @generated from field: google.protobuf.Struct details = 12; + */ + details?: JsonObject | undefined; + + /** + * @generated from field: string error_text = 13; + */ + errorText: string; + + /** + * @generated from field: google.protobuf.Struct tokens_used = 14; + */ + tokensUsed?: JsonObject | undefined; + + /** + * @generated from field: optional int64 latency_ms = 15; + */ + latencyMs?: bigint | undefined; + + /** + * @generated from field: google.protobuf.Timestamp created_at = 16; + */ + createdAt?: Timestamp | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Result. + * Use `create(ResultSchema)` to create a new message. + */ +export declare const ResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.SubmitRunRequest + */ +export declare type SubmitRunRequest = Message<"agentevals.api.v1alpha1.SubmitRunRequest"> & { + /** + * A UUID chosen by the caller, for idempotent retries. Empty means the server picks one. + * + * @generated from field: string run_id = 1; + */ + runId: string; + + /** + * @generated from field: agentevals.api.v1alpha1.RunSpec spec = 2; + */ + spec?: RunSpec | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.SubmitRunRequest. + * Use `create(SubmitRunRequestSchema)` to create a new message. + */ +export declare const SubmitRunRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.SubmitRunResponse + */ +export declare type SubmitRunResponse = Message<"agentevals.api.v1alpha1.SubmitRunResponse"> & { + /** + * @generated from field: agentevals.api.v1alpha1.Run run = 1; + */ + run?: Run | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.SubmitRunResponse. + * Use `create(SubmitRunResponseSchema)` to create a new message. + */ +export declare const SubmitRunResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetRunRequest + */ +export declare type GetRunRequest = Message<"agentevals.api.v1alpha1.GetRunRequest"> & { + /** + * @generated from field: string run_id = 1; + */ + runId: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetRunRequest. + * Use `create(GetRunRequestSchema)` to create a new message. + */ +export declare const GetRunRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetRunResponse + */ +export declare type GetRunResponse = Message<"agentevals.api.v1alpha1.GetRunResponse"> & { + /** + * @generated from field: agentevals.api.v1alpha1.Run run = 1; + */ + run?: Run | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetRunResponse. + * Use `create(GetRunResponseSchema)` to create a new message. + */ +export declare const GetRunResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListRunsRequest + */ +export declare type ListRunsRequest = Message<"agentevals.api.v1alpha1.ListRunsRequest"> & { + /** + * Empty means every status. + * + * @generated from field: repeated agentevals.api.v1alpha1.RunStatus statuses = 1; + */ + statuses: RunStatus[]; + + /** + * @generated from field: agentevals.api.v1alpha1.PageRequest page = 2; + */ + page?: PageRequest | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListRunsRequest. + * Use `create(ListRunsRequestSchema)` to create a new message. + */ +export declare const ListRunsRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListRunsResponse + */ +export declare type ListRunsResponse = Message<"agentevals.api.v1alpha1.ListRunsResponse"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.Run runs = 1; + */ + runs: Run[]; + + /** + * @generated from field: agentevals.api.v1alpha1.PageResponse page = 2; + */ + page?: PageResponse | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListRunsResponse. + * Use `create(ListRunsResponseSchema)` to create a new message. + */ +export declare const ListRunsResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.CancelRunRequest + */ +export declare type CancelRunRequest = Message<"agentevals.api.v1alpha1.CancelRunRequest"> & { + /** + * @generated from field: string run_id = 1; + */ + runId: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.CancelRunRequest. + * Use `create(CancelRunRequestSchema)` to create a new message. + */ +export declare const CancelRunRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.CancelRunResponse + */ +export declare type CancelRunResponse = Message<"agentevals.api.v1alpha1.CancelRunResponse"> & { + /** + * @generated from field: agentevals.api.v1alpha1.Run run = 1; + */ + run?: Run | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.CancelRunResponse. + * Use `create(CancelRunResponseSchema)` to create a new message. + */ +export declare const CancelRunResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListRunResultsRequest + */ +export declare type ListRunResultsRequest = Message<"agentevals.api.v1alpha1.ListRunResultsRequest"> & { + /** + * @generated from field: string run_id = 1; + */ + runId: string; + + /** + * @generated from field: agentevals.api.v1alpha1.PageRequest page = 2; + */ + page?: PageRequest | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListRunResultsRequest. + * Use `create(ListRunResultsRequestSchema)` to create a new message. + */ +export declare const ListRunResultsRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListRunResultsResponse + */ +export declare type ListRunResultsResponse = Message<"agentevals.api.v1alpha1.ListRunResultsResponse"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.Result results = 1; + */ + results: Result[]; + + /** + * @generated from field: agentevals.api.v1alpha1.PageResponse page = 2; + */ + page?: PageResponse | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListRunResultsResponse. + * Use `create(ListRunResultsResponseSchema)` to create a new message. + */ +export declare const ListRunResultsResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.WatchRunRequest + */ +export declare type WatchRunRequest = Message<"agentevals.api.v1alpha1.WatchRunRequest"> & { + /** + * @generated from field: string run_id = 1; + */ + runId: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.WatchRunRequest. + * Use `create(WatchRunRequestSchema)` to create a new message. + */ +export declare const WatchRunRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.WatchRunResponse + */ +export declare type WatchRunResponse = Message<"agentevals.api.v1alpha1.WatchRunResponse"> & { + /** + * The run, sent first and after every change. + * + * @generated from field: agentevals.api.v1alpha1.Run run = 1; + */ + run?: Run | undefined; + + /** + * Results that are new or changed since the previous message. + * + * @generated from field: repeated agentevals.api.v1alpha1.Result results = 2; + */ + results: Result[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.WatchRunResponse. + * Use `create(WatchRunResponseSchema)` to create a new message. + */ +export declare const WatchRunResponseSchema: GenMessage; + +/** + * @generated from enum agentevals.api.v1alpha1.RunStatus + */ +export enum RunStatus { + /** + * @generated from enum value: RUN_STATUS_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: RUN_STATUS_QUEUED = 1; + */ + QUEUED = 1, + + /** + * @generated from enum value: RUN_STATUS_RUNNING = 2; + */ + RUNNING = 2, + + /** + * @generated from enum value: RUN_STATUS_SUCCEEDED = 3; + */ + SUCCEEDED = 3, + + /** + * @generated from enum value: RUN_STATUS_FAILED = 4; + */ + FAILED = 4, + + /** + * @generated from enum value: RUN_STATUS_CANCELLED = 5; + */ + CANCELLED = 5, +} + +/** + * Describes the enum agentevals.api.v1alpha1.RunStatus. + */ +export declare const RunStatusSchema: GenEnum; + +/** + * @generated from enum agentevals.api.v1alpha1.ResultStatus + */ +export enum ResultStatus { + /** + * @generated from enum value: RESULT_STATUS_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: RESULT_STATUS_PASSED = 1; + */ + PASSED = 1, + + /** + * @generated from enum value: RESULT_STATUS_FAILED = 2; + */ + FAILED = 2, + + /** + * @generated from enum value: RESULT_STATUS_ERRORED = 3; + */ + ERRORED = 3, + + /** + * @generated from enum value: RESULT_STATUS_SKIPPED = 4; + */ + SKIPPED = 4, +} + +/** + * Describes the enum agentevals.api.v1alpha1.ResultStatus. + */ +export declare const ResultStatusSchema: GenEnum; + +/** + * @generated from enum agentevals.api.v1alpha1.EvaluatorType + */ +export enum EvaluatorType { + /** + * @generated from enum value: EVALUATOR_TYPE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: EVALUATOR_TYPE_BUILTIN = 1; + */ + BUILTIN = 1, + + /** + * @generated from enum value: EVALUATOR_TYPE_CODE = 2; + */ + CODE = 2, + + /** + * @generated from enum value: EVALUATOR_TYPE_REMOTE = 3; + */ + REMOTE = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.EvaluatorType. + */ +export declare const EvaluatorTypeSchema: GenEnum; + +/** + * @generated from enum agentevals.api.v1alpha1.RunApproach + */ +export enum RunApproach { + /** + * RUN_APPROACH_TRACE_REPLAY. + * + * @generated from enum value: RUN_APPROACH_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * Evaluate recorded traces. + * + * @generated from enum value: RUN_APPROACH_TRACE_REPLAY = 1; + */ + TRACE_REPLAY = 1, +} + +/** + * Describes the enum agentevals.api.v1alpha1.RunApproach. + */ +export declare const RunApproachSchema: GenEnum; + +/** + * Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every + * method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. + * + * @generated from service agentevals.api.v1alpha1.RunService + */ +export declare const RunService: GenService<{ + /** + * Queues a run. Idempotent on run_id: the same id with the same spec returns the existing run; + * a different spec fails with ALREADY_EXISTS and reason RUN_SPEC_CONFLICT. + * + * @generated from rpc agentevals.api.v1alpha1.RunService.SubmitRun + */ + submitRun: { + methodKind: "unary"; + input: typeof SubmitRunRequestSchema; + output: typeof SubmitRunResponseSchema; + }, + /** + * @generated from rpc agentevals.api.v1alpha1.RunService.GetRun + */ + getRun: { + methodKind: "unary"; + input: typeof GetRunRequestSchema; + output: typeof GetRunResponseSchema; + }, + /** + * Newest first. + * + * @generated from rpc agentevals.api.v1alpha1.RunService.ListRuns + */ + listRuns: { + methodKind: "unary"; + input: typeof ListRunsRequestSchema; + output: typeof ListRunsResponseSchema; + }, + /** + * Requests cancellation. Cancelling a finished run changes nothing. + * + * @generated from rpc agentevals.api.v1alpha1.RunService.CancelRun + */ + cancelRun: { + methodKind: "unary"; + input: typeof CancelRunRequestSchema; + output: typeof CancelRunResponseSchema; + }, + /** + * @generated from rpc agentevals.api.v1alpha1.RunService.ListRunResults + */ + listRunResults: { + methodKind: "unary"; + input: typeof ListRunResultsRequestSchema; + output: typeof ListRunResultsResponseSchema; + }, + /** + * The run and its results, then each change, until the run reaches a terminal status. + * + * @generated from rpc agentevals.api.v1alpha1.RunService.WatchRun + */ + watchRun: { + methodKind: "server_streaming"; + input: typeof WatchRunRequestSchema; + output: typeof WatchRunResponseSchema; + }, +}>; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/runs_pb.js b/ui/src/generated/agentevals/api/v1alpha1/runs_pb.js new file mode 100644 index 00000000..0b6d831d --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/runs_pb.js @@ -0,0 +1,205 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/runs.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, serviceDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_agentevals_api_v1alpha1_common } from "./common_pb"; +import { file_agentevals_api_v1alpha1_evaluation } from "./evaluation_pb"; +import { file_google_protobuf_struct, file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/runs.proto. + */ +export const file_agentevals_api_v1alpha1_runs = /*@__PURE__*/ + fileDesc("CiJhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9ydW5zLnByb3RvEhdhZ2VudGV2YWxzLmFwaS52MWFscGhhMSJrCg9IdHRwVHJhY2VUYXJnZXQSEAoIYmFzZV91cmwYASABKAkSEAoIdHJhY2VfaWQYAiABKAkSNAoGZm9ybWF0GAMgASgOMiQuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VGb3JtYXQiOgoOVXBsb2FkZWRUcmFjZXMSEwoLdHJhY2VfY291bnQYASABKAUSEwoLdHJhY2VfZmlsZXMYAiADKAki/gEKC1RyYWNlVGFyZ2V0EjgKBmlubGluZRgBIAEoCzImLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWNlRG9jdW1lbnRIABI2CgRvdGxwGAIgASgLMiYuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuT3RscFRlbGVtZXRyeUgAEjgKBGh0dHAYAyABKAsyKC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5IdHRwVHJhY2VUYXJnZXRIABI7Cgh1cGxvYWRlZBgEIAEoCzInLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlVwbG9hZGVkVHJhY2VzSABCBgoEa2luZCJ/CgpSdW5Db250ZXh0EkEKB2hlYWRlcnMYASADKAsyMC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5SdW5Db250ZXh0LkhlYWRlcnNFbnRyeRouCgxIZWFkZXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASLrAwoHUnVuU3BlYxI2CghhcHByb2FjaBgBIAEoDjIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1bkFwcHJvYWNoEjQKBnRhcmdldBgCIAEoCzIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWNlVGFyZ2V0Ei8KDGFka19ldmFsX3NldBgDIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3RIABI4CgtldmFsX2NvbmZpZxgEIAEoCzIjLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWxQYXJhbXMSJgoFc2lua3MYBSADKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EjQKB2NvbnRleHQYBiABKAsyIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5SdW5Db250ZXh0Ek0KD2NyZWRlbnRpYWxfcmVmcxgHIAMoCzI0LmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1blNwZWMuQ3JlZGVudGlhbFJlZnNFbnRyeRpOChNDcmVkZW50aWFsUmVmc0VudHJ5EgsKA2tleRgBIAEoCRImCgV2YWx1ZRgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3Q6AjgBQgoKCGV2YWxfc2V0IoEDCgNSdW4SDgoGcnVuX2lkGAEgASgJEjIKBnN0YXR1cxgCIAEoDjIiLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1blN0YXR1cxIuCgRzcGVjGAMgASgLMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUnVuU3BlYxIPCgdhdHRlbXB0GAQgASgFEhEKCXdvcmtlcl9pZBgFIAEoCRINCgVlcnJvchgGIAEoCRIoCgdzdW1tYXJ5GAcgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdBIuCgpjcmVhdGVkX2F0GAggASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIuCgpzdGFydGVkX2F0GAkgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIvCgtmaW5pc2hlZF9hdBgKIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASGAoQY2FuY2VsX3JlcXVlc3RlZBgLIAEoCCK4BAoGUmVzdWx0EhEKCXJlc3VsdF9pZBgBIAEoCRIOCgZydW5faWQYAiABKAkSGAoQZXZhbF9zZXRfaXRlbV9pZBgDIAEoCRIaChJldmFsX3NldF9pdGVtX25hbWUYBCABKAkSFgoOZXZhbHVhdG9yX25hbWUYBSABKAkSPgoOZXZhbHVhdG9yX3R5cGUYBiABKA4yJi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0b3JUeXBlEjUKBnN0YXR1cxgHIAEoDjIlLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJlc3VsdFN0YXR1cxISCgVzY29yZRgIIAEoAUgAiAEBEkEKDnBlcl9pbnZvY2F0aW9uGAkgAygLMikuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuSW52b2NhdGlvblJlc3VsdBIQCgh0cmFjZV9pZBgKIAEoCRIPCgdzcGFuX2lkGAsgASgJEigKB2RldGFpbHMYDCABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhIKCmVycm9yX3RleHQYDSABKAkSLAoLdG9rZW5zX3VzZWQYDiABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhcKCmxhdGVuY3lfbXMYDyABKANIAYgBARIuCgpjcmVhdGVkX2F0GBAgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcEIICgZfc2NvcmVCDQoLX2xhdGVuY3lfbXMiUgoQU3VibWl0UnVuUmVxdWVzdBIOCgZydW5faWQYASABKAkSLgoEc3BlYxgCIAEoCzIgLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1blNwZWMiPgoRU3VibWl0UnVuUmVzcG9uc2USKQoDcnVuGAEgASgLMhwuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUnVuIh8KDUdldFJ1blJlcXVlc3QSDgoGcnVuX2lkGAEgASgJIjsKDkdldFJ1blJlc3BvbnNlEikKA3J1bhgBIAEoCzIcLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1biJ7Cg9MaXN0UnVuc1JlcXVlc3QSNAoIc3RhdHVzZXMYASADKA4yIi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5SdW5TdGF0dXMSMgoEcGFnZRgCIAEoCzIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlBhZ2VSZXF1ZXN0InMKEExpc3RSdW5zUmVzcG9uc2USKgoEcnVucxgBIAMoCzIcLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1bhIzCgRwYWdlGAIgASgLMiUuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUGFnZVJlc3BvbnNlIiIKEENhbmNlbFJ1blJlcXVlc3QSDgoGcnVuX2lkGAEgASgJIj4KEUNhbmNlbFJ1blJlc3BvbnNlEikKA3J1bhgBIAEoCzIcLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1biJbChVMaXN0UnVuUmVzdWx0c1JlcXVlc3QSDgoGcnVuX2lkGAEgASgJEjIKBHBhZ2UYAiABKAsyJC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5QYWdlUmVxdWVzdCJ/ChZMaXN0UnVuUmVzdWx0c1Jlc3BvbnNlEjAKB3Jlc3VsdHMYASADKAsyHy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5SZXN1bHQSMwoEcGFnZRgCIAEoCzIlLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlBhZ2VSZXNwb25zZSIhCg9XYXRjaFJ1blJlcXVlc3QSDgoGcnVuX2lkGAEgASgJIm8KEFdhdGNoUnVuUmVzcG9uc2USKQoDcnVuGAEgASgLMhwuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUnVuEjAKB3Jlc3VsdHMYAiADKAsyHy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5SZXN1bHQqoQEKCVJ1blN0YXR1cxIaChZSVU5fU1RBVFVTX1VOU1BFQ0lGSUVEEAASFQoRUlVOX1NUQVRVU19RVUVVRUQQARIWChJSVU5fU1RBVFVTX1JVTk5JTkcQAhIYChRSVU5fU1RBVFVTX1NVQ0NFRURFRBADEhUKEVJVTl9TVEFUVVNfRkFJTEVEEAQSGAoUUlVOX1NUQVRVU19DQU5DRUxMRUQQBSqXAQoMUmVzdWx0U3RhdHVzEh0KGVJFU1VMVF9TVEFUVVNfVU5TUEVDSUZJRUQQABIYChRSRVNVTFRfU1RBVFVTX1BBU1NFRBABEhgKFFJFU1VMVF9TVEFUVVNfRkFJTEVEEAISGQoVUkVTVUxUX1NUQVRVU19FUlJPUkVEEAMSGQoVUkVTVUxUX1NUQVRVU19TS0lQUEVEEAQqfwoNRXZhbHVhdG9yVHlwZRIeChpFVkFMVUFUT1JfVFlQRV9VTlNQRUNJRklFRBAAEhoKFkVWQUxVQVRPUl9UWVBFX0JVSUxUSU4QARIXChNFVkFMVUFUT1JfVFlQRV9DT0RFEAISGQoVRVZBTFVBVE9SX1RZUEVfUkVNT1RFEAMqSgoLUnVuQXBwcm9hY2gSHAoYUlVOX0FQUFJPQUNIX1VOU1BFQ0lGSUVEEAASHQoZUlVOX0FQUFJPQUNIX1RSQUNFX1JFUExBWRABMuYECgpSdW5TZXJ2aWNlEmIKCVN1Ym1pdFJ1bhIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlN1Ym1pdFJ1blJlcXVlc3QaKi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5TdWJtaXRSdW5SZXNwb25zZRJZCgZHZXRSdW4SJi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5HZXRSdW5SZXF1ZXN0GicuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR2V0UnVuUmVzcG9uc2USXwoITGlzdFJ1bnMSKC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5MaXN0UnVuc1JlcXVlc3QaKS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5MaXN0UnVuc1Jlc3BvbnNlEmIKCUNhbmNlbFJ1bhIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkNhbmNlbFJ1blJlcXVlc3QaKi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5DYW5jZWxSdW5SZXNwb25zZRJxCg5MaXN0UnVuUmVzdWx0cxIuLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RSdW5SZXN1bHRzUmVxdWVzdBovLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RSdW5SZXN1bHRzUmVzcG9uc2USYQoIV2F0Y2hSdW4SKC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5XYXRjaFJ1blJlcXVlc3QaKS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5XYXRjaFJ1blJlc3BvbnNlMAFCUVpPZ2l0aHViLmNvbS9hZ2VudGV2YWxzLWRldi9hZ2VudGV2YWxzL2dlbi9nby9hZ2VudGV2YWxzL2FwaS92MWFscGhhMTthcGl2MWFscGhhMWIGcHJvdG8z", [file_agentevals_api_v1alpha1_common, file_agentevals_api_v1alpha1_evaluation, file_google_protobuf_struct, file_google_protobuf_timestamp]); + +/** + * Describes the message agentevals.api.v1alpha1.HttpTraceTarget. + * Use `create(HttpTraceTargetSchema)` to create a new message. + */ +export const HttpTraceTargetSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 0); + +/** + * Describes the message agentevals.api.v1alpha1.UploadedTraces. + * Use `create(UploadedTracesSchema)` to create a new message. + */ +export const UploadedTracesSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 1); + +/** + * Describes the message agentevals.api.v1alpha1.TraceTarget. + * Use `create(TraceTargetSchema)` to create a new message. + */ +export const TraceTargetSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 2); + +/** + * Describes the message agentevals.api.v1alpha1.RunContext. + * Use `create(RunContextSchema)` to create a new message. + */ +export const RunContextSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 3); + +/** + * Describes the message agentevals.api.v1alpha1.RunSpec. + * Use `create(RunSpecSchema)` to create a new message. + */ +export const RunSpecSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 4); + +/** + * Describes the message agentevals.api.v1alpha1.Run. + * Use `create(RunSchema)` to create a new message. + */ +export const RunSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 5); + +/** + * Describes the message agentevals.api.v1alpha1.Result. + * Use `create(ResultSchema)` to create a new message. + */ +export const ResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 6); + +/** + * Describes the message agentevals.api.v1alpha1.SubmitRunRequest. + * Use `create(SubmitRunRequestSchema)` to create a new message. + */ +export const SubmitRunRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 7); + +/** + * Describes the message agentevals.api.v1alpha1.SubmitRunResponse. + * Use `create(SubmitRunResponseSchema)` to create a new message. + */ +export const SubmitRunResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 8); + +/** + * Describes the message agentevals.api.v1alpha1.GetRunRequest. + * Use `create(GetRunRequestSchema)` to create a new message. + */ +export const GetRunRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 9); + +/** + * Describes the message agentevals.api.v1alpha1.GetRunResponse. + * Use `create(GetRunResponseSchema)` to create a new message. + */ +export const GetRunResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 10); + +/** + * Describes the message agentevals.api.v1alpha1.ListRunsRequest. + * Use `create(ListRunsRequestSchema)` to create a new message. + */ +export const ListRunsRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 11); + +/** + * Describes the message agentevals.api.v1alpha1.ListRunsResponse. + * Use `create(ListRunsResponseSchema)` to create a new message. + */ +export const ListRunsResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 12); + +/** + * Describes the message agentevals.api.v1alpha1.CancelRunRequest. + * Use `create(CancelRunRequestSchema)` to create a new message. + */ +export const CancelRunRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 13); + +/** + * Describes the message agentevals.api.v1alpha1.CancelRunResponse. + * Use `create(CancelRunResponseSchema)` to create a new message. + */ +export const CancelRunResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 14); + +/** + * Describes the message agentevals.api.v1alpha1.ListRunResultsRequest. + * Use `create(ListRunResultsRequestSchema)` to create a new message. + */ +export const ListRunResultsRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 15); + +/** + * Describes the message agentevals.api.v1alpha1.ListRunResultsResponse. + * Use `create(ListRunResultsResponseSchema)` to create a new message. + */ +export const ListRunResultsResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 16); + +/** + * Describes the message agentevals.api.v1alpha1.WatchRunRequest. + * Use `create(WatchRunRequestSchema)` to create a new message. + */ +export const WatchRunRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 17); + +/** + * Describes the message agentevals.api.v1alpha1.WatchRunResponse. + * Use `create(WatchRunResponseSchema)` to create a new message. + */ +export const WatchRunResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_runs, 18); + +/** + * Describes the enum agentevals.api.v1alpha1.RunStatus. + */ +export const RunStatusSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_runs, 0); + +/** + * @generated from enum agentevals.api.v1alpha1.RunStatus + */ +export const RunStatus = /*@__PURE__*/ + tsEnum(RunStatusSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.ResultStatus. + */ +export const ResultStatusSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_runs, 1); + +/** + * @generated from enum agentevals.api.v1alpha1.ResultStatus + */ +export const ResultStatus = /*@__PURE__*/ + tsEnum(ResultStatusSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.EvaluatorType. + */ +export const EvaluatorTypeSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_runs, 2); + +/** + * @generated from enum agentevals.api.v1alpha1.EvaluatorType + */ +export const EvaluatorType = /*@__PURE__*/ + tsEnum(EvaluatorTypeSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.RunApproach. + */ +export const RunApproachSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_runs, 3); + +/** + * @generated from enum agentevals.api.v1alpha1.RunApproach + */ +export const RunApproach = /*@__PURE__*/ + tsEnum(RunApproachSchema); + +/** + * Asynchronous evaluation runs, executed by a worker. Needs run storage; without it every + * method fails with FAILED_PRECONDITION and reason STORAGE_NOT_CONFIGURED. + * + * @generated from service agentevals.api.v1alpha1.RunService + */ +export const RunService = /*@__PURE__*/ + serviceDesc(file_agentevals_api_v1alpha1_runs, 0); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.d.ts new file mode 100644 index 00000000..5c691cca --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.d.ts @@ -0,0 +1,212 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/sessions.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { PageRequest, PageResponse } from "./common_pb"; +import type { Session, SessionSummary } from "./views_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/sessions.proto. + */ +export declare const file_agentevals_api_v1alpha1_sessions: GenFile; + +/** + * @generated from message agentevals.api.v1alpha1.ListSessionsRequest + */ +export declare type ListSessionsRequest = Message<"agentevals.api.v1alpha1.ListSessionsRequest"> & { + /** + * @generated from field: agentevals.api.v1alpha1.PageRequest page = 1; + */ + page?: PageRequest | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListSessionsRequest. + * Use `create(ListSessionsRequestSchema)` to create a new message. + */ +export declare const ListSessionsRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ListSessionsResponse + */ +export declare type ListSessionsResponse = Message<"agentevals.api.v1alpha1.ListSessionsResponse"> & { + /** + * @generated from field: repeated agentevals.api.v1alpha1.SessionSummary sessions = 1; + */ + sessions: SessionSummary[]; + + /** + * @generated from field: agentevals.api.v1alpha1.PageResponse page = 2; + */ + page?: PageResponse | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ListSessionsResponse. + * Use `create(ListSessionsResponseSchema)` to create a new message. + */ +export declare const ListSessionsResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetSessionRequest + */ +export declare type GetSessionRequest = Message<"agentevals.api.v1alpha1.GetSessionRequest"> & { + /** + * @generated from field: string id = 1; + */ + id: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetSessionRequest. + * Use `create(GetSessionRequestSchema)` to create a new message. + */ +export declare const GetSessionRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetSessionResponse + */ +export declare type GetSessionResponse = Message<"agentevals.api.v1alpha1.GetSessionResponse"> & { + /** + * @generated from field: agentevals.api.v1alpha1.Session session = 1; + */ + session?: Session | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetSessionResponse. + * Use `create(GetSessionResponseSchema)` to create a new message. + */ +export declare const GetSessionResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.WatchSessionsRequest + */ +export declare type WatchSessionsRequest = Message<"agentevals.api.v1alpha1.WatchSessionsRequest"> & { + /** + * Send each session with its turns. Otherwise upserts carry the summary only. + * + * @generated from field: bool include_turns = 1; + */ + includeTurns: boolean; +}; + +/** + * Describes the message agentevals.api.v1alpha1.WatchSessionsRequest. + * Use `create(WatchSessionsRequestSchema)` to create a new message. + */ +export declare const WatchSessionsRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.SessionRemoved + */ +export declare type SessionRemoved = Message<"agentevals.api.v1alpha1.SessionRemoved"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * The session that took over this session's traces, when the two were merged. + * + * @generated from field: string absorbed_by = 2; + */ + absorbedBy: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.SessionRemoved. + * Use `create(SessionRemovedSchema)` to create a new message. + */ +export declare const SessionRemovedSchema: GenMessage; + +/** + * The end of the initial snapshot. + * + * @generated from message agentevals.api.v1alpha1.Synced + */ +export declare type Synced = Message<"agentevals.api.v1alpha1.Synced"> & { +}; + +/** + * Describes the message agentevals.api.v1alpha1.Synced. + * Use `create(SyncedSchema)` to create a new message. + */ +export declare const SyncedSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.WatchSessionsResponse + */ +export declare type WatchSessionsResponse = Message<"agentevals.api.v1alpha1.WatchSessionsResponse"> & { + /** + * @generated from oneof agentevals.api.v1alpha1.WatchSessionsResponse.event + */ + event: { + /** + * The session's current state. It replaces every earlier upsert with the same id. Upserts + * can be coalesced, so a client sees the latest state, not every intermediate one. + * + * @generated from field: agentevals.api.v1alpha1.Session upsert = 1; + */ + value: Session; + case: "upsert"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.SessionRemoved removed = 2; + */ + value: SessionRemoved; + case: "removed"; + } | { + /** + * @generated from field: agentevals.api.v1alpha1.Synced synced = 3; + */ + value: Synced; + case: "synced"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message agentevals.api.v1alpha1.WatchSessionsResponse. + * Use `create(WatchSessionsResponseSchema)` to create a new message. + */ +export declare const WatchSessionsResponseSchema: GenMessage; + +/** + * Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in + * memory and expire. + * + * @generated from service agentevals.api.v1alpha1.SessionService + */ +export declare const SessionService: GenService<{ + /** + * Sessions newest first, summaries only. + * + * @generated from rpc agentevals.api.v1alpha1.SessionService.ListSessions + */ + listSessions: { + methodKind: "unary"; + input: typeof ListSessionsRequestSchema; + output: typeof ListSessionsResponseSchema; + }, + /** + * @generated from rpc agentevals.api.v1alpha1.SessionService.GetSession + */ + getSession: { + methodKind: "unary"; + input: typeof GetSessionRequestSchema; + output: typeof GetSessionResponseSchema; + }, + /** + * Every current session, then synced, then each change as it happens. + * + * @generated from rpc agentevals.api.v1alpha1.SessionService.WatchSessions + */ + watchSessions: { + methodKind: "server_streaming"; + input: typeof WatchSessionsRequestSchema; + output: typeof WatchSessionsResponseSchema; + }, +}>; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.js b/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.js new file mode 100644 index 00000000..d06c531a --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/sessions_pb.js @@ -0,0 +1,79 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/sessions.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_agentevals_api_v1alpha1_common } from "./common_pb"; +import { file_agentevals_api_v1alpha1_views } from "./views_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/sessions.proto. + */ +export const file_agentevals_api_v1alpha1_sessions = /*@__PURE__*/ + fileDesc("CiZhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9zZXNzaW9ucy5wcm90bxIXYWdlbnRldmFscy5hcGkudjFhbHBoYTEiSQoTTGlzdFNlc3Npb25zUmVxdWVzdBIyCgRwYWdlGAEgASgLMiQuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUGFnZVJlcXVlc3QihgEKFExpc3RTZXNzaW9uc1Jlc3BvbnNlEjkKCHNlc3Npb25zGAEgAygLMicuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuU2Vzc2lvblN1bW1hcnkSMwoEcGFnZRgCIAEoCzIlLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlBhZ2VSZXNwb25zZSIfChFHZXRTZXNzaW9uUmVxdWVzdBIKCgJpZBgBIAEoCSJHChJHZXRTZXNzaW9uUmVzcG9uc2USMQoHc2Vzc2lvbhgBIAEoCzIgLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlNlc3Npb24iLQoUV2F0Y2hTZXNzaW9uc1JlcXVlc3QSFQoNaW5jbHVkZV90dXJucxgBIAEoCCIxCg5TZXNzaW9uUmVtb3ZlZBIKCgJpZBgBIAEoCRITCgthYnNvcmJlZF9ieRgCIAEoCSIICgZTeW5jZWQiwwEKFVdhdGNoU2Vzc2lvbnNSZXNwb25zZRIyCgZ1cHNlcnQYASABKAsyIC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5TZXNzaW9uSAASOgoHcmVtb3ZlZBgCIAEoCzInLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlNlc3Npb25SZW1vdmVkSAASMQoGc3luY2VkGAMgASgLMh8uYWdlbnRldmFscy5hcGkudjFhbHBoYTEuU3luY2VkSABCBwoFZXZlbnQy1gIKDlNlc3Npb25TZXJ2aWNlEmsKDExpc3RTZXNzaW9ucxIsLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RTZXNzaW9uc1JlcXVlc3QaLS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5MaXN0U2Vzc2lvbnNSZXNwb25zZRJlCgpHZXRTZXNzaW9uEiouYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR2V0U2Vzc2lvblJlcXVlc3QaKy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5HZXRTZXNzaW9uUmVzcG9uc2UScAoNV2F0Y2hTZXNzaW9ucxItLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLldhdGNoU2Vzc2lvbnNSZXF1ZXN0Gi4uYWdlbnRldmFscy5hcGkudjFhbHBoYTEuV2F0Y2hTZXNzaW9uc1Jlc3BvbnNlMAFCUVpPZ2l0aHViLmNvbS9hZ2VudGV2YWxzLWRldi9hZ2VudGV2YWxzL2dlbi9nby9hZ2VudGV2YWxzL2FwaS92MWFscGhhMTthcGl2MWFscGhhMWIGcHJvdG8z", [file_agentevals_api_v1alpha1_common, file_agentevals_api_v1alpha1_views]); + +/** + * Describes the message agentevals.api.v1alpha1.ListSessionsRequest. + * Use `create(ListSessionsRequestSchema)` to create a new message. + */ +export const ListSessionsRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 0); + +/** + * Describes the message agentevals.api.v1alpha1.ListSessionsResponse. + * Use `create(ListSessionsResponseSchema)` to create a new message. + */ +export const ListSessionsResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 1); + +/** + * Describes the message agentevals.api.v1alpha1.GetSessionRequest. + * Use `create(GetSessionRequestSchema)` to create a new message. + */ +export const GetSessionRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 2); + +/** + * Describes the message agentevals.api.v1alpha1.GetSessionResponse. + * Use `create(GetSessionResponseSchema)` to create a new message. + */ +export const GetSessionResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 3); + +/** + * Describes the message agentevals.api.v1alpha1.WatchSessionsRequest. + * Use `create(WatchSessionsRequestSchema)` to create a new message. + */ +export const WatchSessionsRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 4); + +/** + * Describes the message agentevals.api.v1alpha1.SessionRemoved. + * Use `create(SessionRemovedSchema)` to create a new message. + */ +export const SessionRemovedSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 5); + +/** + * Describes the message agentevals.api.v1alpha1.Synced. + * Use `create(SyncedSchema)` to create a new message. + */ +export const SyncedSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 6); + +/** + * Describes the message agentevals.api.v1alpha1.WatchSessionsResponse. + * Use `create(WatchSessionsResponseSchema)` to create a new message. + */ +export const WatchSessionsResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_sessions, 7); + +/** + * Live sessions built from the telemetry the OTLP receivers take in. Sessions are kept in + * memory and expire. + * + * @generated from service agentevals.api.v1alpha1.SessionService + */ +export const SessionService = /*@__PURE__*/ + serviceDesc(file_agentevals_api_v1alpha1_sessions, 0); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/system_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/system_pb.d.ts new file mode 100644 index 00000000..ee9028b7 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/system_pb.d.ts @@ -0,0 +1,229 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/system.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { Duration } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/system.proto. + */ +export declare const file_agentevals_api_v1alpha1_system: GenFile; + +/** + * @generated from message agentevals.api.v1alpha1.GetVersionRequest + */ +export declare type GetVersionRequest = Message<"agentevals.api.v1alpha1.GetVersionRequest"> & { +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetVersionRequest. + * Use `create(GetVersionRequestSchema)` to create a new message. + */ +export declare const GetVersionRequestSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetVersionResponse + */ +export declare type GetVersionResponse = Message<"agentevals.api.v1alpha1.GetVersionResponse"> & { + /** + * @generated from field: string version = 1; + */ + version: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetVersionResponse. + * Use `create(GetVersionResponseSchema)` to create a new message. + */ +export declare const GetVersionResponseSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetConfigRequest + */ +export declare type GetConfigRequest = Message<"agentevals.api.v1alpha1.GetConfigRequest"> & { +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetConfigRequest. + * Use `create(GetConfigRequestSchema)` to create a new message. + */ +export declare const GetConfigRequestSchema: GenMessage; + +/** + * Listener ports. 0 when a listener is off. + * + * @generated from message agentevals.api.v1alpha1.Ports + */ +export declare type Ports = Message<"agentevals.api.v1alpha1.Ports"> & { + /** + * @generated from field: int32 http = 1; + */ + http: number; + + /** + * @generated from field: int32 otlp_http = 2; + */ + otlpHttp: number; + + /** + * @generated from field: int32 otlp_grpc = 3; + */ + otlpGrpc: number; + + /** + * @generated from field: int32 api_grpc = 4; + */ + apiGrpc: number; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Ports. + * Use `create(PortsSchema)` to create a new message. + */ +export declare const PortsSchema: GenMessage; + +/** + * Bounds of the live session store. Telemetry over a bound is rejected or evicted. + * + * @generated from message agentevals.api.v1alpha1.LiveLimits + */ +export declare type LiveLimits = Message<"agentevals.api.v1alpha1.LiveLimits"> & { + /** + * @generated from field: int32 max_sessions = 1; + */ + maxSessions: number; + + /** + * @generated from field: int32 spans_per_trace = 2; + */ + spansPerTrace: number; + + /** + * @generated from field: int32 spans_per_session = 3; + */ + spansPerSession: number; + + /** + * @generated from field: int32 traces_per_session = 4; + */ + tracesPerSession: number; + + /** + * @generated from field: int64 max_bytes = 5; + */ + maxBytes: bigint; + + /** + * @generated from field: google.protobuf.Duration completion_grace = 6; + */ + completionGrace?: Duration | undefined; + + /** + * @generated from field: google.protobuf.Duration idle_timeout = 7; + */ + idleTimeout?: Duration | undefined; + + /** + * @generated from field: google.protobuf.Duration session_ttl = 8; + */ + sessionTtl?: Duration | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.LiveLimits. + * Use `create(LiveLimitsSchema)` to create a new message. + */ +export declare const LiveLimitsSchema: GenMessage; + +/** + * Which judge provider API keys the server has in its environment. Values are never returned. + * + * @generated from message agentevals.api.v1alpha1.JudgeKeys + */ +export declare type JudgeKeys = Message<"agentevals.api.v1alpha1.JudgeKeys"> & { + /** + * @generated from field: bool google = 1; + */ + google: boolean; + + /** + * @generated from field: bool anthropic = 2; + */ + anthropic: boolean; + + /** + * @generated from field: bool openai = 3; + */ + openai: boolean; +}; + +/** + * Describes the message agentevals.api.v1alpha1.JudgeKeys. + * Use `create(JudgeKeysSchema)` to create a new message. + */ +export declare const JudgeKeysSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.GetConfigResponse + */ +export declare type GetConfigResponse = Message<"agentevals.api.v1alpha1.GetConfigResponse"> & { + /** + * @generated from field: agentevals.api.v1alpha1.Ports ports = 1; + */ + ports?: Ports | undefined; + + /** + * @generated from field: agentevals.api.v1alpha1.LiveLimits live_limits = 2; + */ + liveLimits?: LiveLimits | undefined; + + /** + * Whether run storage is configured, so RunService works. + * + * @generated from field: bool runs_enabled = 3; + */ + runsEnabled: boolean; + + /** + * Whether evaluation results are emitted as gen_ai.evaluation.result events. + * + * @generated from field: bool evaluation_events_enabled = 4; + */ + evaluationEventsEnabled: boolean; + + /** + * @generated from field: agentevals.api.v1alpha1.JudgeKeys judge_keys = 5; + */ + judgeKeys?: JudgeKeys | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.GetConfigResponse. + * Use `create(GetConfigResponseSchema)` to create a new message. + */ +export declare const GetConfigResponseSchema: GenMessage; + +/** + * @generated from service agentevals.api.v1alpha1.SystemService + */ +export declare const SystemService: GenService<{ + /** + * @generated from rpc agentevals.api.v1alpha1.SystemService.GetVersion + */ + getVersion: { + methodKind: "unary"; + input: typeof GetVersionRequestSchema; + output: typeof GetVersionResponseSchema; + }, + /** + * @generated from rpc agentevals.api.v1alpha1.SystemService.GetConfig + */ + getConfig: { + methodKind: "unary"; + input: typeof GetConfigRequestSchema; + output: typeof GetConfigResponseSchema; + }, +}>; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/system_pb.js b/ui/src/generated/agentevals/api/v1alpha1/system_pb.js new file mode 100644 index 00000000..d47faee7 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/system_pb.js @@ -0,0 +1,68 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/system.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_google_protobuf_duration } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file agentevals/api/v1alpha1/system.proto. + */ +export const file_agentevals_api_v1alpha1_system = /*@__PURE__*/ + fileDesc("CiRhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9zeXN0ZW0ucHJvdG8SF2FnZW50ZXZhbHMuYXBpLnYxYWxwaGExIhMKEUdldFZlcnNpb25SZXF1ZXN0IiUKEkdldFZlcnNpb25SZXNwb25zZRIPCgd2ZXJzaW9uGAEgASgJIhIKEEdldENvbmZpZ1JlcXVlc3QiTQoFUG9ydHMSDAoEaHR0cBgBIAEoBRIRCglvdGxwX2h0dHAYAiABKAUSEQoJb3RscF9ncnBjGAMgASgFEhAKCGFwaV9ncnBjGAQgASgFIpsCCgpMaXZlTGltaXRzEhQKDG1heF9zZXNzaW9ucxgBIAEoBRIXCg9zcGFuc19wZXJfdHJhY2UYAiABKAUSGQoRc3BhbnNfcGVyX3Nlc3Npb24YAyABKAUSGgoSdHJhY2VzX3Blcl9zZXNzaW9uGAQgASgFEhEKCW1heF9ieXRlcxgFIAEoAxIzChBjb21wbGV0aW9uX2dyYWNlGAYgASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9uEi8KDGlkbGVfdGltZW91dBgHIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIuCgtzZXNzaW9uX3R0bBgIIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbiI+CglKdWRnZUtleXMSDgoGZ29vZ2xlGAEgASgIEhEKCWFudGhyb3BpYxgCIAEoCBIOCgZvcGVuYWkYAyABKAgi7QEKEUdldENvbmZpZ1Jlc3BvbnNlEi0KBXBvcnRzGAEgASgLMh4uYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUG9ydHMSOAoLbGl2ZV9saW1pdHMYAiABKAsyIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5MaXZlTGltaXRzEhQKDHJ1bnNfZW5hYmxlZBgDIAEoCBIhChlldmFsdWF0aW9uX2V2ZW50c19lbmFibGVkGAQgASgIEjYKCmp1ZGdlX2tleXMYBSABKAsyIi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5KdWRnZUtleXMy2gEKDVN5c3RlbVNlcnZpY2USZQoKR2V0VmVyc2lvbhIqLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkdldFZlcnNpb25SZXF1ZXN0GisuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR2V0VmVyc2lvblJlc3BvbnNlEmIKCUdldENvbmZpZxIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkdldENvbmZpZ1JlcXVlc3QaKi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5HZXRDb25maWdSZXNwb25zZUJRWk9naXRodWIuY29tL2FnZW50ZXZhbHMtZGV2L2FnZW50ZXZhbHMvZ2VuL2dvL2FnZW50ZXZhbHMvYXBpL3YxYWxwaGExO2FwaXYxYWxwaGExYgZwcm90bzM", [file_google_protobuf_duration]); + +/** + * Describes the message agentevals.api.v1alpha1.GetVersionRequest. + * Use `create(GetVersionRequestSchema)` to create a new message. + */ +export const GetVersionRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 0); + +/** + * Describes the message agentevals.api.v1alpha1.GetVersionResponse. + * Use `create(GetVersionResponseSchema)` to create a new message. + */ +export const GetVersionResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 1); + +/** + * Describes the message agentevals.api.v1alpha1.GetConfigRequest. + * Use `create(GetConfigRequestSchema)` to create a new message. + */ +export const GetConfigRequestSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 2); + +/** + * Describes the message agentevals.api.v1alpha1.Ports. + * Use `create(PortsSchema)` to create a new message. + */ +export const PortsSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 3); + +/** + * Describes the message agentevals.api.v1alpha1.LiveLimits. + * Use `create(LiveLimitsSchema)` to create a new message. + */ +export const LiveLimitsSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 4); + +/** + * Describes the message agentevals.api.v1alpha1.JudgeKeys. + * Use `create(JudgeKeysSchema)` to create a new message. + */ +export const JudgeKeysSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 5); + +/** + * Describes the message agentevals.api.v1alpha1.GetConfigResponse. + * Use `create(GetConfigResponseSchema)` to create a new message. + */ +export const GetConfigResponseSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_system, 6); + +/** + * @generated from service agentevals.api.v1alpha1.SystemService + */ +export const SystemService = /*@__PURE__*/ + serviceDesc(file_agentevals_api_v1alpha1_system, 0); + diff --git a/ui/src/generated/agentevals/api/v1alpha1/views_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/views_pb.d.ts new file mode 100644 index 00000000..b87e1fc6 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/views_pb.d.ts @@ -0,0 +1,737 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/views.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { Duration, Timestamp, Value } from "@bufbuild/protobuf/wkt"; +import type { Status_StatusCode } from "../../../opentelemetry/proto/trace/v1/trace_pb"; +import type { Resource } from "../../../opentelemetry/proto/resource/v1/resource_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/views.proto. + */ +export declare const file_agentevals_api_v1alpha1_views: GenFile; + +/** + * Token counts. Unset means not recorded; 0 means zero. + * + * @generated from message agentevals.api.v1alpha1.Usage + */ +export declare type Usage = Message<"agentevals.api.v1alpha1.Usage"> & { + /** + * gen_ai.usage.input_tokens, which includes cached input tokens. + * + * @generated from field: optional int64 input_tokens = 1; + */ + inputTokens?: bigint | undefined; + + /** + * gen_ai.usage.output_tokens. + * + * @generated from field: optional int64 output_tokens = 2; + */ + outputTokens?: bigint | undefined; + + /** + * gen_ai.usage.cache_read.input_tokens. + * + * @generated from field: optional int64 cache_read_input_tokens = 3; + */ + cacheReadInputTokens?: bigint | undefined; + + /** + * gen_ai.usage.cache_write.input_tokens. + * + * @generated from field: optional int64 cache_write_input_tokens = 4; + */ + cacheWriteInputTokens?: bigint | undefined; + + /** + * gen_ai.usage.reasoning.output_tokens. + * + * @generated from field: optional int64 reasoning_output_tokens = 5; + */ + reasoningOutputTokens?: bigint | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Usage. + * Use `create(UsageSchema)` to create a new message. + */ +export declare const UsageSchema: GenMessage; + +/** + * Why something is missing or approximate. Codes are stable; reasons are for people and never + * carry message content. + * + * @generated from message agentevals.api.v1alpha1.Gap + */ +export declare type Gap = Message<"agentevals.api.v1alpha1.Gap"> & { + /** + * @generated from field: string code = 1; + */ + code: string; + + /** + * @generated from field: string reason = 2; + */ + reason: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Gap. + * Use `create(GapSchema)` to create a new message. + */ +export declare const GapSchema: GenMessage; + +/** + * An evaluation result with the fields of the gen_ai.evaluation.result event. + * + * @generated from message agentevals.api.v1alpha1.EvaluationResult + */ +export declare type EvaluationResult = Message<"agentevals.api.v1alpha1.EvaluationResult"> & { + /** + * gen_ai.evaluation.name. + * + * @generated from field: string name = 1; + */ + name: string; + + /** + * gen_ai.evaluation.score.value. + * + * @generated from field: optional double score_value = 2; + */ + scoreValue?: number | undefined; + + /** + * gen_ai.evaluation.score.label. + * + * @generated from field: optional string score_label = 3; + */ + scoreLabel?: string | undefined; + + /** + * gen_ai.evaluation.explanation. + * + * @generated from field: optional string explanation = 4; + */ + explanation?: string | undefined; + + /** + * error.type, when the evaluation itself failed. + * + * @generated from field: optional string error_type = 5; + */ + errorType?: string | undefined; + + /** + * The judged span. + * + * @generated from field: string trace_id = 6; + */ + traceId: string; + + /** + * @generated from field: string span_id = 7; + */ + spanId: string; + + /** + * gen_ai.response.id of the judged response. + * + * @generated from field: string response_id = 8; + */ + responseId: string; + + /** + * service.name of the producer, for results found in telemetry. + * + * @generated from field: string source_service = 9; + */ + sourceService: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.EvaluationResult. + * Use `create(EvaluationResultSchema)` to create a new message. + */ +export declare const EvaluationResultSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.ToolCallView + */ +export declare type ToolCallView = Message<"agentevals.api.v1alpha1.ToolCallView"> & { + /** + * The outermost span of the call. Empty when the call is only known from message parts. + * + * @generated from field: string span_id = 1; + */ + spanId: string; + + /** + * gen_ai.tool.name. + * + * @generated from field: string name = 2; + */ + name: string; + + /** + * gen_ai.tool.call.id. + * + * @generated from field: optional string call_id = 3; + */ + callId?: string | undefined; + + /** + * gen_ai.tool.call.arguments, parsed. Unset when not recorded. + * + * @generated from field: google.protobuf.Value arguments = 4; + */ + arguments?: Value | undefined; + + /** + * gen_ai.tool.call.result, parsed, paired with the call by call id and then by order. Unset + * when not recorded. + * + * @generated from field: google.protobuf.Value result = 5; + */ + result?: Value | undefined; + + /** + * @generated from field: bool is_error = 6; + */ + isError: boolean; + + /** + * error.type. + * + * @generated from field: string error_type = 7; + */ + errorType: string; + + /** + * @generated from field: google.protobuf.Timestamp started_at = 8; + */ + startedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Duration duration = 9; + */ + duration?: Duration | undefined; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ToolCallView. + * Use `create(ToolCallViewSchema)` to create a new message. + */ +export declare const ToolCallViewSchema: GenMessage; + +/** + * One logical model call: a leaf inference span with its single leaf wrappers. + * + * @generated from message agentevals.api.v1alpha1.ModelCallView + */ +export declare type ModelCallView = Message<"agentevals.api.v1alpha1.ModelCallView"> & { + /** + * @generated from field: string span_id = 1; + */ + spanId: string; + + /** + * gen_ai.operation.name. + * + * @generated from field: string operation = 2; + */ + operation: string; + + /** + * gen_ai.provider.name. + * + * @generated from field: string provider = 3; + */ + provider: string; + + /** + * gen_ai.request.model. + * + * @generated from field: string request_model = 4; + */ + requestModel: string; + + /** + * gen_ai.response.model. + * + * @generated from field: string response_model = 5; + */ + responseModel: string; + + /** + * gen_ai.response.id. + * + * @generated from field: string response_id = 6; + */ + responseId: string; + + /** + * @generated from field: agentevals.api.v1alpha1.Usage usage = 7; + */ + usage?: Usage | undefined; + + /** + * gen_ai.response.finish_reasons. + * + * @generated from field: repeated string finish_reasons = 8; + */ + finishReasons: string[]; + + /** + * The span status. + * + * @generated from field: opentelemetry.proto.trace.v1.Status.StatusCode status = 9; + */ + status: Status_StatusCode; + + /** + * error.type. + * + * @generated from field: string error_type = 10; + */ + errorType: string; + + /** + * @generated from field: google.protobuf.Timestamp started_at = 11; + */ + startedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Duration duration = 12; + */ + duration?: Duration | undefined; + + /** + * gen_ai.agent.name of the agent that made the call. + * + * @generated from field: string agent_name = 13; + */ + agentName: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ModelCallView. + * Use `create(ModelCallViewSchema)` to create a new message. + */ +export declare const ModelCallViewSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.TurnView + */ +export declare type TurnView = Message<"agentevals.api.v1alpha1.TurnView"> & { + /** + * Position in the session or conversation, from 0. + * + * @generated from field: int32 index = 1; + */ + index: number; + + /** + * The turn's anchor: an agent or workflow span, or the trace root. + * + * @generated from field: string trace_id = 2; + */ + traceId: string; + + /** + * @generated from field: string span_id = 3; + */ + spanId: string; + + /** + * The anchor's span status. + * + * @generated from field: opentelemetry.proto.trace.v1.Status.StatusCode status = 4; + */ + status: Status_StatusCode; + + /** + * error.type. + * + * @generated from field: string error_type = 5; + */ + errorType: string; + + /** + * @generated from field: google.protobuf.Timestamp started_at = 6; + */ + startedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Duration duration = 7; + */ + duration?: Duration | undefined; + + /** + * gen_ai.agent.name of the anchor. + * + * @generated from field: string agent_name = 8; + */ + agentName: string; + + /** + * The user message that started the turn (gen_ai.input.messages). Unset when not recorded. + * + * @generated from field: optional string user_text = 9; + */ + userText?: string | undefined; + + /** + * The final answer (gen_ai.output.messages). Unset when not recorded. + * + * @generated from field: optional string answer_text = 10; + */ + answerText?: string | undefined; + + /** + * Whether message content was recorded for this turn. + * + * @generated from field: bool content_captured = 11; + */ + contentCaptured: boolean; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.ToolCallView tool_calls = 12; + */ + toolCalls: ToolCallView[]; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.ModelCallView model_calls = 13; + */ + modelCalls: ModelCallView[]; + + /** + * Sum over model_calls. + * + * @generated from field: agentevals.api.v1alpha1.Usage usage = 14; + */ + usage?: Usage | undefined; + + /** + * gen_ai.evaluation.result events other producers attached to this turn's spans. + * + * @generated from field: repeated agentevals.api.v1alpha1.EvaluationResult external_evaluations = 15; + */ + externalEvaluations: EvaluationResult[]; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.Gap gaps = 16; + */ + gaps: Gap[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.TurnView. + * Use `create(TurnViewSchema)` to create a new message. + */ +export declare const TurnViewSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.SessionTotals + */ +export declare type SessionTotals = Message<"agentevals.api.v1alpha1.SessionTotals"> & { + /** + * @generated from field: int32 turns = 1; + */ + turns: number; + + /** + * Turns whose status is ERROR. + * + * @generated from field: int32 failed_turns = 2; + */ + failedTurns: number; + + /** + * @generated from field: int32 model_calls = 3; + */ + modelCalls: number; + + /** + * @generated from field: int32 tool_calls = 4; + */ + toolCalls: number; + + /** + * Sum over all model calls. + * + * @generated from field: agentevals.api.v1alpha1.Usage usage = 5; + */ + usage?: Usage | undefined; + + /** + * Model calls that recorded token usage, for "measured in N of M". + * + * @generated from field: int32 model_calls_with_usage = 6; + */ + modelCallsWithUsage: number; + + /** + * Tool calls that recorded arguments. + * + * @generated from field: int32 tool_calls_with_arguments = 7; + */ + toolCallsWithArguments: number; +}; + +/** + * Describes the message agentevals.api.v1alpha1.SessionTotals. + * Use `create(SessionTotalsSchema)` to create a new message. + */ +export declare const SessionTotalsSchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.SessionSummary + */ +export declare type SessionSummary = Message<"agentevals.api.v1alpha1.SessionSummary"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * The last 6 characters of a UUID or hex id, which are often time ordered so their first + * characters collide; any other id, such as a session name, as it is. + * + * @generated from field: string short_id = 2; + */ + shortId: string; + + /** + * The resource's gen_ai.agent.name, else the first turn's agent, else the resource's + * service.name. + * + * @generated from field: string agent = 3; + */ + agent: string; + + /** + * The resource's gen_ai.agent.version, else the first turn's, else service.version. + * + * @generated from field: string agent_version = 4; + */ + agentVersion: string; + + /** + * @generated from field: agentevals.api.v1alpha1.SessionState state = 5; + */ + state: SessionState; + + /** + * @generated from field: google.protobuf.Timestamp started_at = 6; + */ + startedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Timestamp ended_at = 7; + */ + endedAt?: Timestamp | undefined; + + /** + * @generated from field: google.protobuf.Duration duration = 8; + */ + duration?: Duration | undefined; + + /** + * The first turn's user text. Unset when not recorded. + * + * @generated from field: optional string first_user_text = 9; + */ + firstUserText?: string | undefined; + + /** + * @generated from field: agentevals.api.v1alpha1.ContentCapture content_capture = 10; + */ + contentCapture: ContentCapture; + + /** + * @generated from field: agentevals.api.v1alpha1.SessionTotals totals = 11; + */ + totals?: SessionTotals | undefined; + + /** + * gen_ai.response.model of each call, else gen_ai.request.model, distinct, in first use + * order. + * + * @generated from field: repeated string models = 12; + */ + models: string[]; + + /** + * The resource of the service that produced the session. + * + * @generated from field: opentelemetry.proto.resource.v1.Resource resource = 13; + */ + resource?: Resource | undefined; + + /** + * gen_ai.conversation.id. + * + * @generated from field: string conversation_id = 14; + */ + conversationId: string; + + /** + * @generated from field: repeated string trace_ids = 15; + */ + traceIds: string[]; + + /** + * The eval set the session was recorded for, when the producer named one. + * + * @generated from field: string eval_set_id = 16; + */ + evalSetId: string; + + /** + * @generated from field: int32 span_count = 17; + */ + spanCount: number; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.Gap gaps = 18; + */ + gaps: Gap[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.SessionSummary. + * Use `create(SessionSummarySchema)` to create a new message. + */ +export declare const SessionSummarySchema: GenMessage; + +/** + * @generated from message agentevals.api.v1alpha1.Session + */ +export declare type Session = Message<"agentevals.api.v1alpha1.Session"> & { + /** + * @generated from field: agentevals.api.v1alpha1.SessionSummary summary = 1; + */ + summary?: SessionSummary | undefined; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.TurnView turns = 2; + */ + turns: TurnView[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Session. + * Use `create(SessionSchema)` to create a new message. + */ +export declare const SessionSchema: GenMessage; + +/** + * Traces that form one evaluation unit, as conversation turns. + * + * @generated from message agentevals.api.v1alpha1.ConversationView + */ +export declare type ConversationView = Message<"agentevals.api.v1alpha1.ConversationView"> & { + /** + * The unit's key: its earliest trace id, or its conversation key. + * + * @generated from field: string key = 1; + */ + key: string; + + /** + * gen_ai.conversation.id. + * + * @generated from field: string conversation_id = 2; + */ + conversationId: string; + + /** + * @generated from field: repeated string trace_ids = 3; + */ + traceIds: string[]; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.TurnView turns = 4; + */ + turns: TurnView[]; + + /** + * @generated from field: repeated agentevals.api.v1alpha1.Gap gaps = 5; + */ + gaps: Gap[]; +}; + +/** + * Describes the message agentevals.api.v1alpha1.ConversationView. + * Use `create(ConversationViewSchema)` to create a new message. + */ +export declare const ConversationViewSchema: GenMessage; + +/** + * @generated from enum agentevals.api.v1alpha1.SessionState + */ +export enum SessionState { + /** + * @generated from enum value: SESSION_STATE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * Spans may still arrive. + * + * @generated from enum value: SESSION_STATE_LIVE = 1; + */ + LIVE = 1, + + /** + * The completion grace or the idle timeout has passed. New spans reopen the session. + * + * @generated from enum value: SESSION_STATE_COMPLETE = 2; + */ + COMPLETE = 2, +} + +/** + * Describes the enum agentevals.api.v1alpha1.SessionState. + */ +export declare const SessionStateSchema: GenEnum; + +/** + * @generated from enum agentevals.api.v1alpha1.ContentCapture + */ +export enum ContentCapture { + /** + * No turns yet. + * + * @generated from enum value: CONTENT_CAPTURE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: CONTENT_CAPTURE_ON = 1; + */ + ON = 1, + + /** + * @generated from enum value: CONTENT_CAPTURE_OFF = 2; + */ + OFF = 2, + + /** + * Some turns have content, others do not. + * + * @generated from enum value: CONTENT_CAPTURE_PARTIAL = 3; + */ + PARTIAL = 3, +} + +/** + * Describes the enum agentevals.api.v1alpha1.ContentCapture. + */ +export declare const ContentCaptureSchema: GenEnum; + diff --git a/ui/src/generated/agentevals/api/v1alpha1/views_pb.js b/ui/src/generated/agentevals/api/v1alpha1/views_pb.js new file mode 100644 index 00000000..f81dd239 --- /dev/null +++ b/ui/src/generated/agentevals/api/v1alpha1/views_pb.js @@ -0,0 +1,109 @@ +// @generated by protoc-gen-es v2.13.0 +// @generated from file agentevals/api/v1alpha1/views.proto (package agentevals.api.v1alpha1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_google_protobuf_duration, file_google_protobuf_struct, file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt"; +import { file_opentelemetry_proto_resource_v1_resource } from "../../../opentelemetry/proto/resource/v1/resource_pb"; +import { file_opentelemetry_proto_trace_v1_trace } from "../../../opentelemetry/proto/trace/v1/trace_pb"; + +/** + * Describes the file agentevals/api/v1alpha1/views.proto. + */ +export const file_agentevals_api_v1alpha1_views = /*@__PURE__*/ + fileDesc("CiNhZ2VudGV2YWxzL2FwaS92MWFscGhhMS92aWV3cy5wcm90bxIXYWdlbnRldmFscy5hcGkudjFhbHBoYTEiqQIKBVVzYWdlEhkKDGlucHV0X3Rva2VucxgBIAEoA0gAiAEBEhoKDW91dHB1dF90b2tlbnMYAiABKANIAYgBARIkChdjYWNoZV9yZWFkX2lucHV0X3Rva2VucxgDIAEoA0gCiAEBEiUKGGNhY2hlX3dyaXRlX2lucHV0X3Rva2VucxgEIAEoA0gDiAEBEiQKF3JlYXNvbmluZ19vdXRwdXRfdG9rZW5zGAUgASgDSASIAQFCDwoNX2lucHV0X3Rva2Vuc0IQCg5fb3V0cHV0X3Rva2Vuc0IaChhfY2FjaGVfcmVhZF9pbnB1dF90b2tlbnNCGwoZX2NhY2hlX3dyaXRlX2lucHV0X3Rva2Vuc0IaChhfcmVhc29uaW5nX291dHB1dF90b2tlbnMiIwoDR2FwEgwKBGNvZGUYASABKAkSDgoGcmVhc29uGAIgASgJIpYCChBFdmFsdWF0aW9uUmVzdWx0EgwKBG5hbWUYASABKAkSGAoLc2NvcmVfdmFsdWUYAiABKAFIAIgBARIYCgtzY29yZV9sYWJlbBgDIAEoCUgBiAEBEhgKC2V4cGxhbmF0aW9uGAQgASgJSAKIAQESFwoKZXJyb3JfdHlwZRgFIAEoCUgDiAEBEhAKCHRyYWNlX2lkGAYgASgJEg8KB3NwYW5faWQYByABKAkSEwoLcmVzcG9uc2VfaWQYCCABKAkSFgoOc291cmNlX3NlcnZpY2UYCSABKAlCDgoMX3Njb3JlX3ZhbHVlQg4KDF9zY29yZV9sYWJlbEIOCgxfZXhwbGFuYXRpb25CDQoLX2Vycm9yX3R5cGUipQIKDFRvb2xDYWxsVmlldxIPCgdzcGFuX2lkGAEgASgJEgwKBG5hbWUYAiABKAkSFAoHY2FsbF9pZBgDIAEoCUgAiAEBEikKCWFyZ3VtZW50cxgEIAEoCzIWLmdvb2dsZS5wcm90b2J1Zi5WYWx1ZRImCgZyZXN1bHQYBSABKAsyFi5nb29nbGUucHJvdG9idWYuVmFsdWUSEAoIaXNfZXJyb3IYBiABKAgSEgoKZXJyb3JfdHlwZRgHIAEoCRIuCgpzdGFydGVkX2F0GAggASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIrCghkdXJhdGlvbhgJIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbkIKCghfY2FsbF9pZCKWAwoNTW9kZWxDYWxsVmlldxIPCgdzcGFuX2lkGAEgASgJEhEKCW9wZXJhdGlvbhgCIAEoCRIQCghwcm92aWRlchgDIAEoCRIVCg1yZXF1ZXN0X21vZGVsGAQgASgJEhYKDnJlc3BvbnNlX21vZGVsGAUgASgJEhMKC3Jlc3BvbnNlX2lkGAYgASgJEi0KBXVzYWdlGAcgASgLMh4uYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVXNhZ2USFgoOZmluaXNoX3JlYXNvbnMYCCADKAkSPwoGc3RhdHVzGAkgASgOMi8ub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TdGF0dXMuU3RhdHVzQ29kZRISCgplcnJvcl90eXBlGAogASgJEi4KCnN0YXJ0ZWRfYXQYCyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEisKCGR1cmF0aW9uGAwgASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9uEhIKCmFnZW50X25hbWUYDSABKAkiiAUKCFR1cm5WaWV3Eg0KBWluZGV4GAEgASgFEhAKCHRyYWNlX2lkGAIgASgJEg8KB3NwYW5faWQYAyABKAkSPwoGc3RhdHVzGAQgASgOMi8ub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TdGF0dXMuU3RhdHVzQ29kZRISCgplcnJvcl90eXBlGAUgASgJEi4KCnN0YXJ0ZWRfYXQYBiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEisKCGR1cmF0aW9uGAcgASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9uEhIKCmFnZW50X25hbWUYCCABKAkSFgoJdXNlcl90ZXh0GAkgASgJSACIAQESGAoLYW5zd2VyX3RleHQYCiABKAlIAYgBARIYChBjb250ZW50X2NhcHR1cmVkGAsgASgIEjkKCnRvb2xfY2FsbHMYDCADKAsyJS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Ub29sQ2FsbFZpZXcSOwoLbW9kZWxfY2FsbHMYDSADKAsyJi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Nb2RlbENhbGxWaWV3Ei0KBXVzYWdlGA4gASgLMh4uYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVXNhZ2USRwoUZXh0ZXJuYWxfZXZhbHVhdGlvbnMYDyADKAsyKS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0aW9uUmVzdWx0EioKBGdhcHMYECADKAsyHC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5HYXBCDAoKX3VzZXJfdGV4dEIOCgxfYW5zd2VyX3RleHQizwEKDVNlc3Npb25Ub3RhbHMSDQoFdHVybnMYASABKAUSFAoMZmFpbGVkX3R1cm5zGAIgASgFEhMKC21vZGVsX2NhbGxzGAMgASgFEhIKCnRvb2xfY2FsbHMYBCABKAUSLQoFdXNhZ2UYBSABKAsyHi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Vc2FnZRIeChZtb2RlbF9jYWxsc193aXRoX3VzYWdlGAYgASgFEiEKGXRvb2xfY2FsbHNfd2l0aF9hcmd1bWVudHMYByABKAUijwUKDlNlc3Npb25TdW1tYXJ5EgoKAmlkGAEgASgJEhAKCHNob3J0X2lkGAIgASgJEg0KBWFnZW50GAMgASgJEhUKDWFnZW50X3ZlcnNpb24YBCABKAkSNAoFc3RhdGUYBSABKA4yJS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5TZXNzaW9uU3RhdGUSLgoKc3RhcnRlZF9hdBgGIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kZWRfYXQYByABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEisKCGR1cmF0aW9uGAggASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9uEhwKD2ZpcnN0X3VzZXJfdGV4dBgJIAEoCUgAiAEBEkAKD2NvbnRlbnRfY2FwdHVyZRgKIAEoDjInLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkNvbnRlbnRDYXB0dXJlEjYKBnRvdGFscxgLIAEoCzImLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlNlc3Npb25Ub3RhbHMSDgoGbW9kZWxzGAwgAygJEjsKCHJlc291cmNlGA0gASgLMikub3BlbnRlbGVtZXRyeS5wcm90by5yZXNvdXJjZS52MS5SZXNvdXJjZRIXCg9jb252ZXJzYXRpb25faWQYDiABKAkSEQoJdHJhY2VfaWRzGA8gAygJEhMKC2V2YWxfc2V0X2lkGBAgASgJEhIKCnNwYW5fY291bnQYESABKAUSKgoEZ2FwcxgSIAMoCzIcLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkdhcEISChBfZmlyc3RfdXNlcl90ZXh0InUKB1Nlc3Npb24SOAoHc3VtbWFyeRgBIAEoCzInLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlNlc3Npb25TdW1tYXJ5EjAKBXR1cm5zGAIgAygLMiEuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHVyblZpZXciqQEKEENvbnZlcnNhdGlvblZpZXcSCwoDa2V5GAEgASgJEhcKD2NvbnZlcnNhdGlvbl9pZBgCIAEoCRIRCgl0cmFjZV9pZHMYAyADKAkSMAoFdHVybnMYBCADKAsyIS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UdXJuVmlldxIqCgRnYXBzGAUgAygLMhwuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR2FwKmEKDFNlc3Npb25TdGF0ZRIdChlTRVNTSU9OX1NUQVRFX1VOU1BFQ0lGSUVEEAASFgoSU0VTU0lPTl9TVEFURV9MSVZFEAESGgoWU0VTU0lPTl9TVEFURV9DT01QTEVURRACKn8KDkNvbnRlbnRDYXB0dXJlEh8KG0NPTlRFTlRfQ0FQVFVSRV9VTlNQRUNJRklFRBAAEhYKEkNPTlRFTlRfQ0FQVFVSRV9PThABEhcKE0NPTlRFTlRfQ0FQVFVSRV9PRkYQAhIbChdDT05URU5UX0NBUFRVUkVfUEFSVElBTBADQlFaT2dpdGh1Yi5jb20vYWdlbnRldmFscy1kZXYvYWdlbnRldmFscy9nZW4vZ28vYWdlbnRldmFscy9hcGkvdjFhbHBoYTE7YXBpdjFhbHBoYTFiBnByb3RvMw", [file_google_protobuf_duration, file_google_protobuf_struct, file_google_protobuf_timestamp, file_opentelemetry_proto_resource_v1_resource, file_opentelemetry_proto_trace_v1_trace]); + +/** + * Describes the message agentevals.api.v1alpha1.Usage. + * Use `create(UsageSchema)` to create a new message. + */ +export const UsageSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 0); + +/** + * Describes the message agentevals.api.v1alpha1.Gap. + * Use `create(GapSchema)` to create a new message. + */ +export const GapSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 1); + +/** + * Describes the message agentevals.api.v1alpha1.EvaluationResult. + * Use `create(EvaluationResultSchema)` to create a new message. + */ +export const EvaluationResultSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 2); + +/** + * Describes the message agentevals.api.v1alpha1.ToolCallView. + * Use `create(ToolCallViewSchema)` to create a new message. + */ +export const ToolCallViewSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 3); + +/** + * Describes the message agentevals.api.v1alpha1.ModelCallView. + * Use `create(ModelCallViewSchema)` to create a new message. + */ +export const ModelCallViewSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 4); + +/** + * Describes the message agentevals.api.v1alpha1.TurnView. + * Use `create(TurnViewSchema)` to create a new message. + */ +export const TurnViewSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 5); + +/** + * Describes the message agentevals.api.v1alpha1.SessionTotals. + * Use `create(SessionTotalsSchema)` to create a new message. + */ +export const SessionTotalsSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 6); + +/** + * Describes the message agentevals.api.v1alpha1.SessionSummary. + * Use `create(SessionSummarySchema)` to create a new message. + */ +export const SessionSummarySchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 7); + +/** + * Describes the message agentevals.api.v1alpha1.Session. + * Use `create(SessionSchema)` to create a new message. + */ +export const SessionSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 8); + +/** + * Describes the message agentevals.api.v1alpha1.ConversationView. + * Use `create(ConversationViewSchema)` to create a new message. + */ +export const ConversationViewSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_views, 9); + +/** + * Describes the enum agentevals.api.v1alpha1.SessionState. + */ +export const SessionStateSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_views, 0); + +/** + * @generated from enum agentevals.api.v1alpha1.SessionState + */ +export const SessionState = /*@__PURE__*/ + tsEnum(SessionStateSchema); + +/** + * Describes the enum agentevals.api.v1alpha1.ContentCapture. + */ +export const ContentCaptureSchema = /*@__PURE__*/ + enumDesc(file_agentevals_api_v1alpha1_views, 1); + +/** + * @generated from enum agentevals.api.v1alpha1.ContentCapture + */ +export const ContentCapture = /*@__PURE__*/ + tsEnum(ContentCaptureSchema); + diff --git a/ui/src/generated/opentelemetry/proto/common/v1/common_pb.d.ts b/ui/src/generated/opentelemetry/proto/common/v1/common_pb.d.ts new file mode 100644 index 00000000..dc622a69 --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/common/v1/common_pb.d.ts @@ -0,0 +1,278 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/common/v1/common.proto (package opentelemetry.proto.common.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file opentelemetry/proto/common/v1/common.proto. + */ +export declare const file_opentelemetry_proto_common_v1_common: GenFile; + +/** + * Represents any type of attribute value. AnyValue may contain a + * primitive value such as a string or integer or it may contain an arbitrary nested + * object containing arrays, key-value lists and primitives. + * + * @generated from message opentelemetry.proto.common.v1.AnyValue + */ +export declare type AnyValue = Message<"opentelemetry.proto.common.v1.AnyValue"> & { + /** + * The value is one of the listed fields. It is valid for all values to be unspecified + * in which case this AnyValue is considered to be "empty". + * + * @generated from oneof opentelemetry.proto.common.v1.AnyValue.value + */ + value: { + /** + * @generated from field: string string_value = 1; + */ + value: string; + case: "stringValue"; + } | { + /** + * @generated from field: bool bool_value = 2; + */ + value: boolean; + case: "boolValue"; + } | { + /** + * @generated from field: int64 int_value = 3; + */ + value: bigint; + case: "intValue"; + } | { + /** + * @generated from field: double double_value = 4; + */ + value: number; + case: "doubleValue"; + } | { + /** + * @generated from field: opentelemetry.proto.common.v1.ArrayValue array_value = 5; + */ + value: ArrayValue; + case: "arrayValue"; + } | { + /** + * @generated from field: opentelemetry.proto.common.v1.KeyValueList kvlist_value = 6; + */ + value: KeyValueList; + case: "kvlistValue"; + } | { + /** + * @generated from field: bytes bytes_value = 7; + */ + value: Uint8Array; + case: "bytesValue"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.AnyValue. + * Use `create(AnyValueSchema)` to create a new message. + */ +export declare const AnyValueSchema: GenMessage; + +/** + * ArrayValue is a list of AnyValue messages. We need ArrayValue as a message + * since oneof in AnyValue does not allow repeated fields. + * + * @generated from message opentelemetry.proto.common.v1.ArrayValue + */ +export declare type ArrayValue = Message<"opentelemetry.proto.common.v1.ArrayValue"> & { + /** + * Array of values. The array may be empty (contain 0 elements). + * + * @generated from field: repeated opentelemetry.proto.common.v1.AnyValue values = 1; + */ + values: AnyValue[]; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.ArrayValue. + * Use `create(ArrayValueSchema)` to create a new message. + */ +export declare const ArrayValueSchema: GenMessage; + +/** + * KeyValueList is a list of KeyValue messages. We need KeyValueList as a message + * since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need + * a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to + * avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches + * are semantically equivalent. + * + * @generated from message opentelemetry.proto.common.v1.KeyValueList + */ +export declare type KeyValueList = Message<"opentelemetry.proto.common.v1.KeyValueList"> & { + /** + * A collection of key/value pairs of key-value pairs. The list may be empty (may + * contain 0 elements). + * + * The keys MUST be unique (it is not allowed to have more than one + * value with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue values = 1; + */ + values: KeyValue[]; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.KeyValueList. + * Use `create(KeyValueListSchema)` to create a new message. + */ +export declare const KeyValueListSchema: GenMessage; + +/** + * Represents a key-value pair that is used to store Span attributes, Link + * attributes, etc. + * + * @generated from message opentelemetry.proto.common.v1.KeyValue + */ +export declare type KeyValue = Message<"opentelemetry.proto.common.v1.KeyValue"> & { + /** + * The key name of the pair. + * + * @generated from field: string key = 1; + */ + key: string; + + /** + * The value of the pair. + * + * @generated from field: opentelemetry.proto.common.v1.AnyValue value = 2; + */ + value?: AnyValue | undefined; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.KeyValue. + * Use `create(KeyValueSchema)` to create a new message. + */ +export declare const KeyValueSchema: GenMessage; + +/** + * InstrumentationScope is a message representing the instrumentation scope information + * such as the fully qualified name and version. + * + * @generated from message opentelemetry.proto.common.v1.InstrumentationScope + */ +export declare type InstrumentationScope = Message<"opentelemetry.proto.common.v1.InstrumentationScope"> & { + /** + * A name denoting the Instrumentation scope. + * An empty instrumentation scope name means the name is unknown. + * + * @generated from field: string name = 1; + */ + name: string; + + /** + * Defines the version of the instrumentation scope. + * An empty instrumentation scope version means the version is unknown. + * + * @generated from field: string version = 2; + */ + version: string; + + /** + * Additional attributes that describe the scope. [Optional]. + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + */ + attributes: KeyValue[]; + + /** + * The number of attributes that were discarded. Attributes + * can be discarded because their keys are too long or because there are too many + * attributes. If this value is 0, then no attributes were dropped. + * + * @generated from field: uint32 dropped_attributes_count = 4; + */ + droppedAttributesCount: number; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.InstrumentationScope. + * Use `create(InstrumentationScopeSchema)` to create a new message. + */ +export declare const InstrumentationScopeSchema: GenMessage; + +/** + * A reference to an Entity. + * Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. + * + * Status: [Development] + * + * @generated from message opentelemetry.proto.common.v1.EntityRef + */ +export declare type EntityRef = Message<"opentelemetry.proto.common.v1.EntityRef"> & { + /** + * The Schema URL, if known. This is the identifier of the Schema that the entity data + * is recorded in. To learn more about Schema URL see + * https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + * + * This schema_url applies to the data in this message and to the Resource attributes + * referenced by id_keys and description_keys. + * TODO: discuss if we are happy with this somewhat complicated definition of what + * the schema_url applies to. + * + * This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. + * + * @generated from field: string schema_url = 1; + */ + schemaUrl: string; + + /** + * Defines the type of the entity. MUST not change during the lifetime of the entity. + * For example: "service" or "host". This field is required and MUST not be empty + * for valid entities. + * + * @generated from field: string type = 2; + */ + type: string; + + /** + * Attribute Keys that identify the entity. + * MUST not change during the lifetime of the entity. The Id must contain at least one attribute. + * These keys MUST exist in the containing {message}.attributes. + * + * @generated from field: repeated string id_keys = 3; + */ + idKeys: string[]; + + /** + * Descriptive (non-identifying) attribute keys of the entity. + * MAY change over the lifetime of the entity. MAY be empty. + * These attribute keys are not part of entity's identity. + * These keys MUST exist in the containing {message}.attributes. + * + * @generated from field: repeated string description_keys = 4; + */ + descriptionKeys: string[]; +}; + +/** + * Describes the message opentelemetry.proto.common.v1.EntityRef. + * Use `create(EntityRefSchema)` to create a new message. + */ +export declare const EntityRefSchema: GenMessage; + diff --git a/ui/src/generated/opentelemetry/proto/common/v1/common_pb.js b/ui/src/generated/opentelemetry/proto/common/v1/common_pb.js new file mode 100644 index 00000000..8d6081dc --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/common/v1/common_pb.js @@ -0,0 +1,68 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/common/v1/common.proto (package opentelemetry.proto.common.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; + +/** + * Describes the file opentelemetry/proto/common/v1/common.proto. + */ +export const file_opentelemetry_proto_common_v1_common = /*@__PURE__*/ + fileDesc("CipvcGVudGVsZW1ldHJ5L3Byb3RvL2NvbW1vbi92MS9jb21tb24ucHJvdG8SHW9wZW50ZWxlbWV0cnkucHJvdG8uY29tbW9uLnYxIowCCghBbnlWYWx1ZRIWCgxzdHJpbmdfdmFsdWUYASABKAlIABIUCgpib29sX3ZhbHVlGAIgASgISAASEwoJaW50X3ZhbHVlGAMgASgDSAASFgoMZG91YmxlX3ZhbHVlGAQgASgBSAASQAoLYXJyYXlfdmFsdWUYBSABKAsyKS5vcGVudGVsZW1ldHJ5LnByb3RvLmNvbW1vbi52MS5BcnJheVZhbHVlSAASQwoMa3ZsaXN0X3ZhbHVlGAYgASgLMisub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuS2V5VmFsdWVMaXN0SAASFQoLYnl0ZXNfdmFsdWUYByABKAxIAEIHCgV2YWx1ZSJFCgpBcnJheVZhbHVlEjcKBnZhbHVlcxgBIAMoCzInLm9wZW50ZWxlbWV0cnkucHJvdG8uY29tbW9uLnYxLkFueVZhbHVlIkcKDEtleVZhbHVlTGlzdBI3CgZ2YWx1ZXMYASADKAsyJy5vcGVudGVsZW1ldHJ5LnByb3RvLmNvbW1vbi52MS5LZXlWYWx1ZSJPCghLZXlWYWx1ZRILCgNrZXkYASABKAkSNgoFdmFsdWUYAiABKAsyJy5vcGVudGVsZW1ldHJ5LnByb3RvLmNvbW1vbi52MS5BbnlWYWx1ZSKUAQoUSW5zdHJ1bWVudGF0aW9uU2NvcGUSDAoEbmFtZRgBIAEoCRIPCgd2ZXJzaW9uGAIgASgJEjsKCmF0dHJpYnV0ZXMYAyADKAsyJy5vcGVudGVsZW1ldHJ5LnByb3RvLmNvbW1vbi52MS5LZXlWYWx1ZRIgChhkcm9wcGVkX2F0dHJpYnV0ZXNfY291bnQYBCABKA0iWAoJRW50aXR5UmVmEhIKCnNjaGVtYV91cmwYASABKAkSDAoEdHlwZRgCIAEoCRIPCgdpZF9rZXlzGAMgAygJEhgKEGRlc2NyaXB0aW9uX2tleXMYBCADKAlCewogaW8ub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjFCC0NvbW1vblByb3RvUAFaKGdvLm9wZW50ZWxlbWV0cnkuaW8vcHJvdG8vb3RscC9jb21tb24vdjGqAh1PcGVuVGVsZW1ldHJ5LlByb3RvLkNvbW1vbi5WMWIGcHJvdG8z"); + +/** + * Describes the message opentelemetry.proto.common.v1.AnyValue. + * Use `create(AnyValueSchema)` to create a new message. + */ +export const AnyValueSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 0); + +/** + * Describes the message opentelemetry.proto.common.v1.ArrayValue. + * Use `create(ArrayValueSchema)` to create a new message. + */ +export const ArrayValueSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 1); + +/** + * Describes the message opentelemetry.proto.common.v1.KeyValueList. + * Use `create(KeyValueListSchema)` to create a new message. + */ +export const KeyValueListSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 2); + +/** + * Describes the message opentelemetry.proto.common.v1.KeyValue. + * Use `create(KeyValueSchema)` to create a new message. + */ +export const KeyValueSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 3); + +/** + * Describes the message opentelemetry.proto.common.v1.InstrumentationScope. + * Use `create(InstrumentationScopeSchema)` to create a new message. + */ +export const InstrumentationScopeSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 4); + +/** + * Describes the message opentelemetry.proto.common.v1.EntityRef. + * Use `create(EntityRefSchema)` to create a new message. + */ +export const EntityRefSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_common_v1_common, 5); + diff --git a/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.d.ts b/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.d.ts new file mode 100644 index 00000000..269bba23 --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.d.ts @@ -0,0 +1,461 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/logs/v1/logs.proto (package opentelemetry.proto.logs.v1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { Resource } from "../../resource/v1/resource_pb"; +import type { AnyValue, InstrumentationScope, KeyValue } from "../../common/v1/common_pb"; + +/** + * Describes the file opentelemetry/proto/logs/v1/logs.proto. + */ +export declare const file_opentelemetry_proto_logs_v1_logs: GenFile; + +/** + * LogsData represents the logs data that can be stored in a persistent storage, + * OR can be embedded by other protocols that transfer OTLP logs data but do not + * implement the OTLP protocol. + * + * The main difference between this message and collector protocol is that + * in this message there will not be any "control" or "metadata" specific to + * OTLP protocol. + * + * When new fields are added into this message, the OTLP request MUST be updated + * as well. + * + * @generated from message opentelemetry.proto.logs.v1.LogsData + */ +export declare type LogsData = Message<"opentelemetry.proto.logs.v1.LogsData"> & { + /** + * An array of ResourceLogs. + * For data coming from a single resource this array will typically contain + * one element. Intermediary nodes that receive data from multiple origins + * typically batch the data before forwarding further and in that case this + * array will contain multiple elements. + * + * @generated from field: repeated opentelemetry.proto.logs.v1.ResourceLogs resource_logs = 1; + */ + resourceLogs: ResourceLogs[]; +}; + +/** + * Describes the message opentelemetry.proto.logs.v1.LogsData. + * Use `create(LogsDataSchema)` to create a new message. + */ +export declare const LogsDataSchema: GenMessage; + +/** + * A collection of ScopeLogs from a Resource. + * + * @generated from message opentelemetry.proto.logs.v1.ResourceLogs + */ +export declare type ResourceLogs = Message<"opentelemetry.proto.logs.v1.ResourceLogs"> & { + /** + * The resource for the logs in this message. + * If this field is not set then resource info is unknown. + * + * @generated from field: opentelemetry.proto.resource.v1.Resource resource = 1; + */ + resource?: Resource | undefined; + + /** + * A list of ScopeLogs that originate from a resource. + * + * @generated from field: repeated opentelemetry.proto.logs.v1.ScopeLogs scope_logs = 2; + */ + scopeLogs: ScopeLogs[]; + + /** + * The Schema URL, if known. This is the identifier of the Schema that the resource data + * is recorded in. Notably, the last part of the URL path is the version number of the + * schema: http[s]://server[:port]/path/. To learn more about Schema URL see + * https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + * This schema_url applies to the data in the "resource" field. It does not apply + * to the data in the "scope_logs" field which have their own schema_url field. + * + * @generated from field: string schema_url = 3; + */ + schemaUrl: string; +}; + +/** + * Describes the message opentelemetry.proto.logs.v1.ResourceLogs. + * Use `create(ResourceLogsSchema)` to create a new message. + */ +export declare const ResourceLogsSchema: GenMessage; + +/** + * A collection of Logs produced by a Scope. + * + * @generated from message opentelemetry.proto.logs.v1.ScopeLogs + */ +export declare type ScopeLogs = Message<"opentelemetry.proto.logs.v1.ScopeLogs"> & { + /** + * The instrumentation scope information for the logs in this message. + * Semantically when InstrumentationScope isn't set, it is equivalent with + * an empty instrumentation scope name (unknown). + * + * @generated from field: opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + */ + scope?: InstrumentationScope | undefined; + + /** + * A list of log records. + * + * @generated from field: repeated opentelemetry.proto.logs.v1.LogRecord log_records = 2; + */ + logRecords: LogRecord[]; + + /** + * The Schema URL, if known. This is the identifier of the Schema that the log data + * is recorded in. Notably, the last part of the URL path is the version number of the + * schema: http[s]://server[:port]/path/. To learn more about Schema URL see + * https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + * This schema_url applies to the data in the "scope" field and all logs in the + * "log_records" field. + * + * @generated from field: string schema_url = 3; + */ + schemaUrl: string; +}; + +/** + * Describes the message opentelemetry.proto.logs.v1.ScopeLogs. + * Use `create(ScopeLogsSchema)` to create a new message. + */ +export declare const ScopeLogsSchema: GenMessage; + +/** + * A log record according to OpenTelemetry Log Data Model: + * https://github.com/open-telemetry/oteps/blob/main/text/logs/0097-log-data-model.md + * + * @generated from message opentelemetry.proto.logs.v1.LogRecord + */ +export declare type LogRecord = Message<"opentelemetry.proto.logs.v1.LogRecord"> & { + /** + * time_unix_nano is the time when the event occurred. + * Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + * Value of 0 indicates unknown or missing timestamp. + * + * @generated from field: fixed64 time_unix_nano = 1; + */ + timeUnixNano: bigint; + + /** + * Time when the event was observed by the collection system. + * For events that originate in OpenTelemetry (e.g. using OpenTelemetry Logging SDK) + * this timestamp is typically set at the generation time and is equal to Timestamp. + * For events originating externally and collected by OpenTelemetry (e.g. using + * Collector) this is the time when OpenTelemetry's code observed the event measured + * by the clock of the OpenTelemetry code. This field MUST be set once the event is + * observed by OpenTelemetry. + * + * For converting OpenTelemetry log data to formats that support only one timestamp or + * when receiving OpenTelemetry log data by recipients that support only one timestamp + * internally the following logic is recommended: + * - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. + * + * Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + * Value of 0 indicates unknown or missing timestamp. + * + * @generated from field: fixed64 observed_time_unix_nano = 11; + */ + observedTimeUnixNano: bigint; + + /** + * Numerical value of the severity, normalized to values described in Log Data Model. + * [Optional]. + * + * @generated from field: opentelemetry.proto.logs.v1.SeverityNumber severity_number = 2; + */ + severityNumber: SeverityNumber; + + /** + * The severity text (also known as log level). The original string representation as + * it is known at the source. [Optional]. + * + * @generated from field: string severity_text = 3; + */ + severityText: string; + + /** + * A value containing the body of the log record. Can be for example a human-readable + * string message (including multi-line) describing the event in a free form or it can + * be a structured data composed of arrays and maps of other values. [Optional]. + * + * @generated from field: opentelemetry.proto.common.v1.AnyValue body = 5; + */ + body?: AnyValue | undefined; + + /** + * Additional attributes that describe the specific event occurrence. [Optional]. + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; + */ + attributes: KeyValue[]; + + /** + * @generated from field: uint32 dropped_attributes_count = 7; + */ + droppedAttributesCount: number; + + /** + * Flags, a bit field. 8 least significant bits are the trace flags as + * defined in W3C Trace Context specification. 24 most significant bits are reserved + * and must be set to 0. Readers must not assume that 24 most significant bits + * will be zero and must correctly mask the bits when reading 8-bit trace flag (use + * flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK). [Optional]. + * + * @generated from field: fixed32 flags = 8; + */ + flags: number; + + /** + * A unique identifier for a trace. All logs from the same trace share + * the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + * of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + * is zero-length and thus is also invalid). + * + * This field is optional. + * + * The receivers SHOULD assume that the log record is not associated with a + * trace if any of the following is true: + * - the field is not present, + * - the field contains an invalid value. + * + * @generated from field: bytes trace_id = 9; + */ + traceId: Uint8Array; + + /** + * A unique identifier for a span within a trace, assigned when the span + * is created. The ID is an 8-byte array. An ID with all zeroes OR of length + * other than 8 bytes is considered invalid (empty string in OTLP/JSON + * is zero-length and thus is also invalid). + * + * This field is optional. If the sender specifies a valid span_id then it SHOULD also + * specify a valid trace_id. + * + * The receivers SHOULD assume that the log record is not associated with a + * span if any of the following is true: + * - the field is not present, + * - the field contains an invalid value. + * + * @generated from field: bytes span_id = 10; + */ + spanId: Uint8Array; + + /** + * A unique identifier of event category/type. + * All events with the same event_name are expected to conform to the same + * schema for both their attributes and their body. + * + * Recommended to be fully qualified and short (no longer than 256 characters). + * + * Presence of event_name on the log record identifies this record + * as an event. + * + * [Optional]. + * + * @generated from field: string event_name = 12; + */ + eventName: string; +}; + +/** + * Describes the message opentelemetry.proto.logs.v1.LogRecord. + * Use `create(LogRecordSchema)` to create a new message. + */ +export declare const LogRecordSchema: GenMessage; + +/** + * Possible values for LogRecord.SeverityNumber. + * + * @generated from enum opentelemetry.proto.logs.v1.SeverityNumber + */ +export enum SeverityNumber { + /** + * UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. + * + * @generated from enum value: SEVERITY_NUMBER_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: SEVERITY_NUMBER_TRACE = 1; + */ + TRACE = 1, + + /** + * @generated from enum value: SEVERITY_NUMBER_TRACE2 = 2; + */ + TRACE2 = 2, + + /** + * @generated from enum value: SEVERITY_NUMBER_TRACE3 = 3; + */ + TRACE3 = 3, + + /** + * @generated from enum value: SEVERITY_NUMBER_TRACE4 = 4; + */ + TRACE4 = 4, + + /** + * @generated from enum value: SEVERITY_NUMBER_DEBUG = 5; + */ + DEBUG = 5, + + /** + * @generated from enum value: SEVERITY_NUMBER_DEBUG2 = 6; + */ + DEBUG2 = 6, + + /** + * @generated from enum value: SEVERITY_NUMBER_DEBUG3 = 7; + */ + DEBUG3 = 7, + + /** + * @generated from enum value: SEVERITY_NUMBER_DEBUG4 = 8; + */ + DEBUG4 = 8, + + /** + * @generated from enum value: SEVERITY_NUMBER_INFO = 9; + */ + INFO = 9, + + /** + * @generated from enum value: SEVERITY_NUMBER_INFO2 = 10; + */ + INFO2 = 10, + + /** + * @generated from enum value: SEVERITY_NUMBER_INFO3 = 11; + */ + INFO3 = 11, + + /** + * @generated from enum value: SEVERITY_NUMBER_INFO4 = 12; + */ + INFO4 = 12, + + /** + * @generated from enum value: SEVERITY_NUMBER_WARN = 13; + */ + WARN = 13, + + /** + * @generated from enum value: SEVERITY_NUMBER_WARN2 = 14; + */ + WARN2 = 14, + + /** + * @generated from enum value: SEVERITY_NUMBER_WARN3 = 15; + */ + WARN3 = 15, + + /** + * @generated from enum value: SEVERITY_NUMBER_WARN4 = 16; + */ + WARN4 = 16, + + /** + * @generated from enum value: SEVERITY_NUMBER_ERROR = 17; + */ + ERROR = 17, + + /** + * @generated from enum value: SEVERITY_NUMBER_ERROR2 = 18; + */ + ERROR2 = 18, + + /** + * @generated from enum value: SEVERITY_NUMBER_ERROR3 = 19; + */ + ERROR3 = 19, + + /** + * @generated from enum value: SEVERITY_NUMBER_ERROR4 = 20; + */ + ERROR4 = 20, + + /** + * @generated from enum value: SEVERITY_NUMBER_FATAL = 21; + */ + FATAL = 21, + + /** + * @generated from enum value: SEVERITY_NUMBER_FATAL2 = 22; + */ + FATAL2 = 22, + + /** + * @generated from enum value: SEVERITY_NUMBER_FATAL3 = 23; + */ + FATAL3 = 23, + + /** + * @generated from enum value: SEVERITY_NUMBER_FATAL4 = 24; + */ + FATAL4 = 24, +} + +/** + * Describes the enum opentelemetry.proto.logs.v1.SeverityNumber. + */ +export declare const SeverityNumberSchema: GenEnum; + +/** + * LogRecordFlags represents constants used to interpret the + * LogRecord.flags field, which is protobuf 'fixed32' type and is to + * be used as bit-fields. Each non-zero value defined in this enum is + * a bit-mask. To extract the bit-field, for example, use an + * expression like: + * + * (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) + * + * + * @generated from enum opentelemetry.proto.logs.v1.LogRecordFlags + */ +export enum LogRecordFlags { + /** + * The zero value for the enum. Should not be used for comparisons. + * Instead use bitwise "and" with the appropriate mask as shown above. + * + * @generated from enum value: LOG_RECORD_FLAGS_DO_NOT_USE = 0; + */ + DO_NOT_USE = 0, + + /** + * Bits 0-7 are used for trace flags. + * + * @generated from enum value: LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 255; + */ + TRACE_FLAGS_MASK = 255, +} + +/** + * Describes the enum opentelemetry.proto.logs.v1.LogRecordFlags. + */ +export declare const LogRecordFlagsSchema: GenEnum; + diff --git a/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.js b/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.js new file mode 100644 index 00000000..6bf1df7a --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/logs/v1/logs_pb.js @@ -0,0 +1,91 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/logs/v1/logs.proto (package opentelemetry.proto.logs.v1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_opentelemetry_proto_common_v1_common } from "../../common/v1/common_pb"; +import { file_opentelemetry_proto_resource_v1_resource } from "../../resource/v1/resource_pb"; + +/** + * Describes the file opentelemetry/proto/logs/v1/logs.proto. + */ +export const file_opentelemetry_proto_logs_v1_logs = /*@__PURE__*/ + fileDesc("CiZvcGVudGVsZW1ldHJ5L3Byb3RvL2xvZ3MvdjEvbG9ncy5wcm90bxIbb3BlbnRlbGVtZXRyeS5wcm90by5sb2dzLnYxIkwKCExvZ3NEYXRhEkAKDXJlc291cmNlX2xvZ3MYASADKAsyKS5vcGVudGVsZW1ldHJ5LnByb3RvLmxvZ3MudjEuUmVzb3VyY2VMb2dzIqMBCgxSZXNvdXJjZUxvZ3MSOwoIcmVzb3VyY2UYASABKAsyKS5vcGVudGVsZW1ldHJ5LnByb3RvLnJlc291cmNlLnYxLlJlc291cmNlEjoKCnNjb3BlX2xvZ3MYAiADKAsyJi5vcGVudGVsZW1ldHJ5LnByb3RvLmxvZ3MudjEuU2NvcGVMb2dzEhIKCnNjaGVtYV91cmwYAyABKAlKBgjoBxDpByKgAQoJU2NvcGVMb2dzEkIKBXNjb3BlGAEgASgLMjMub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuSW5zdHJ1bWVudGF0aW9uU2NvcGUSOwoLbG9nX3JlY29yZHMYAiADKAsyJi5vcGVudGVsZW1ldHJ5LnByb3RvLmxvZ3MudjEuTG9nUmVjb3JkEhIKCnNjaGVtYV91cmwYAyABKAkigwMKCUxvZ1JlY29yZBIWCg50aW1lX3VuaXhfbmFubxgBIAEoBhIfChdvYnNlcnZlZF90aW1lX3VuaXhfbmFubxgLIAEoBhJECg9zZXZlcml0eV9udW1iZXIYAiABKA4yKy5vcGVudGVsZW1ldHJ5LnByb3RvLmxvZ3MudjEuU2V2ZXJpdHlOdW1iZXISFQoNc2V2ZXJpdHlfdGV4dBgDIAEoCRI1CgRib2R5GAUgASgLMicub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuQW55VmFsdWUSOwoKYXR0cmlidXRlcxgGIAMoCzInLm9wZW50ZWxlbWV0cnkucHJvdG8uY29tbW9uLnYxLktleVZhbHVlEiAKGGRyb3BwZWRfYXR0cmlidXRlc19jb3VudBgHIAEoDRINCgVmbGFncxgIIAEoBxIQCgh0cmFjZV9pZBgJIAEoDBIPCgdzcGFuX2lkGAogASgMEhIKCmV2ZW50X25hbWUYDCABKAlKBAgEEAUqwwUKDlNldmVyaXR5TnVtYmVyEh8KG1NFVkVSSVRZX05VTUJFUl9VTlNQRUNJRklFRBAAEhkKFVNFVkVSSVRZX05VTUJFUl9UUkFDRRABEhoKFlNFVkVSSVRZX05VTUJFUl9UUkFDRTIQAhIaChZTRVZFUklUWV9OVU1CRVJfVFJBQ0UzEAMSGgoWU0VWRVJJVFlfTlVNQkVSX1RSQUNFNBAEEhkKFVNFVkVSSVRZX05VTUJFUl9ERUJVRxAFEhoKFlNFVkVSSVRZX05VTUJFUl9ERUJVRzIQBhIaChZTRVZFUklUWV9OVU1CRVJfREVCVUczEAcSGgoWU0VWRVJJVFlfTlVNQkVSX0RFQlVHNBAIEhgKFFNFVkVSSVRZX05VTUJFUl9JTkZPEAkSGQoVU0VWRVJJVFlfTlVNQkVSX0lORk8yEAoSGQoVU0VWRVJJVFlfTlVNQkVSX0lORk8zEAsSGQoVU0VWRVJJVFlfTlVNQkVSX0lORk80EAwSGAoUU0VWRVJJVFlfTlVNQkVSX1dBUk4QDRIZChVTRVZFUklUWV9OVU1CRVJfV0FSTjIQDhIZChVTRVZFUklUWV9OVU1CRVJfV0FSTjMQDxIZChVTRVZFUklUWV9OVU1CRVJfV0FSTjQQEBIZChVTRVZFUklUWV9OVU1CRVJfRVJST1IQERIaChZTRVZFUklUWV9OVU1CRVJfRVJST1IyEBISGgoWU0VWRVJJVFlfTlVNQkVSX0VSUk9SMxATEhoKFlNFVkVSSVRZX05VTUJFUl9FUlJPUjQQFBIZChVTRVZFUklUWV9OVU1CRVJfRkFUQUwQFRIaChZTRVZFUklUWV9OVU1CRVJfRkFUQUwyEBYSGgoWU0VWRVJJVFlfTlVNQkVSX0ZBVEFMMxAXEhoKFlNFVkVSSVRZX05VTUJFUl9GQVRBTDQQGCpZCg5Mb2dSZWNvcmRGbGFncxIfChtMT0dfUkVDT1JEX0ZMQUdTX0RPX05PVF9VU0UQABImCiFMT0dfUkVDT1JEX0ZMQUdTX1RSQUNFX0ZMQUdTX01BU0sQ/wFCcwoeaW8ub3BlbnRlbGVtZXRyeS5wcm90by5sb2dzLnYxQglMb2dzUHJvdG9QAVomZ28ub3BlbnRlbGVtZXRyeS5pby9wcm90by9vdGxwL2xvZ3MvdjGqAhtPcGVuVGVsZW1ldHJ5LlByb3RvLkxvZ3MuVjFiBnByb3RvMw", [file_opentelemetry_proto_common_v1_common, file_opentelemetry_proto_resource_v1_resource]); + +/** + * Describes the message opentelemetry.proto.logs.v1.LogsData. + * Use `create(LogsDataSchema)` to create a new message. + */ +export const LogsDataSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_logs_v1_logs, 0); + +/** + * Describes the message opentelemetry.proto.logs.v1.ResourceLogs. + * Use `create(ResourceLogsSchema)` to create a new message. + */ +export const ResourceLogsSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_logs_v1_logs, 1); + +/** + * Describes the message opentelemetry.proto.logs.v1.ScopeLogs. + * Use `create(ScopeLogsSchema)` to create a new message. + */ +export const ScopeLogsSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_logs_v1_logs, 2); + +/** + * Describes the message opentelemetry.proto.logs.v1.LogRecord. + * Use `create(LogRecordSchema)` to create a new message. + */ +export const LogRecordSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_logs_v1_logs, 3); + +/** + * Describes the enum opentelemetry.proto.logs.v1.SeverityNumber. + */ +export const SeverityNumberSchema = /*@__PURE__*/ + enumDesc(file_opentelemetry_proto_logs_v1_logs, 0); + +/** + * Possible values for LogRecord.SeverityNumber. + * + * @generated from enum opentelemetry.proto.logs.v1.SeverityNumber + */ +export const SeverityNumber = /*@__PURE__*/ + tsEnum(SeverityNumberSchema); + +/** + * Describes the enum opentelemetry.proto.logs.v1.LogRecordFlags. + */ +export const LogRecordFlagsSchema = /*@__PURE__*/ + enumDesc(file_opentelemetry_proto_logs_v1_logs, 1); + +/** + * LogRecordFlags represents constants used to interpret the + * LogRecord.flags field, which is protobuf 'fixed32' type and is to + * be used as bit-fields. Each non-zero value defined in this enum is + * a bit-mask. To extract the bit-field, for example, use an + * expression like: + * + * (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) + * + * + * @generated from enum opentelemetry.proto.logs.v1.LogRecordFlags + */ +export const LogRecordFlags = /*@__PURE__*/ + tsEnum(LogRecordFlagsSchema); + diff --git a/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.d.ts b/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.d.ts new file mode 100644 index 00000000..41eeba16 --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.d.ts @@ -0,0 +1,69 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/resource/v1/resource.proto (package opentelemetry.proto.resource.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { EntityRef, KeyValue } from "../../common/v1/common_pb"; + +/** + * Describes the file opentelemetry/proto/resource/v1/resource.proto. + */ +export declare const file_opentelemetry_proto_resource_v1_resource: GenFile; + +/** + * Resource information. + * + * @generated from message opentelemetry.proto.resource.v1.Resource + */ +export declare type Resource = Message<"opentelemetry.proto.resource.v1.Resource"> & { + /** + * Set of attributes that describe the resource. + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + */ + attributes: KeyValue[]; + + /** + * The number of dropped attributes. If the value is 0, then + * no attributes were dropped. + * + * @generated from field: uint32 dropped_attributes_count = 2; + */ + droppedAttributesCount: number; + + /** + * Set of entities that participate in this Resource. + * + * Note: keys in the references MUST exist in attributes of this message. + * + * Status: [Development] + * + * @generated from field: repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; + */ + entityRefs: EntityRef[]; +}; + +/** + * Describes the message opentelemetry.proto.resource.v1.Resource. + * Use `create(ResourceSchema)` to create a new message. + */ +export declare const ResourceSchema: GenMessage; + diff --git a/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.js b/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.js new file mode 100644 index 00000000..4855caf6 --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/resource/v1/resource_pb.js @@ -0,0 +1,34 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/resource/v1/resource.proto (package opentelemetry.proto.resource.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_opentelemetry_proto_common_v1_common } from "../../common/v1/common_pb"; + +/** + * Describes the file opentelemetry/proto/resource/v1/resource.proto. + */ +export const file_opentelemetry_proto_resource_v1_resource = /*@__PURE__*/ + fileDesc("Ci5vcGVudGVsZW1ldHJ5L3Byb3RvL3Jlc291cmNlL3YxL3Jlc291cmNlLnByb3RvEh9vcGVudGVsZW1ldHJ5LnByb3RvLnJlc291cmNlLnYxIqgBCghSZXNvdXJjZRI7CgphdHRyaWJ1dGVzGAEgAygLMicub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuS2V5VmFsdWUSIAoYZHJvcHBlZF9hdHRyaWJ1dGVzX2NvdW50GAIgASgNEj0KC2VudGl0eV9yZWZzGAMgAygLMigub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuRW50aXR5UmVmQoMBCiJpby5vcGVudGVsZW1ldHJ5LnByb3RvLnJlc291cmNlLnYxQg1SZXNvdXJjZVByb3RvUAFaKmdvLm9wZW50ZWxlbWV0cnkuaW8vcHJvdG8vb3RscC9yZXNvdXJjZS92MaoCH09wZW5UZWxlbWV0cnkuUHJvdG8uUmVzb3VyY2UuVjFiBnByb3RvMw", [file_opentelemetry_proto_common_v1_common]); + +/** + * Describes the message opentelemetry.proto.resource.v1.Resource. + * Use `create(ResourceSchema)` to create a new message. + */ +export const ResourceSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_resource_v1_resource, 0); + diff --git a/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.d.ts b/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.d.ts new file mode 100644 index 00000000..1e62b4d7 --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.d.ts @@ -0,0 +1,646 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/trace/v1/trace.proto (package opentelemetry.proto.trace.v1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; +import type { Resource } from "../../resource/v1/resource_pb"; +import type { InstrumentationScope, KeyValue } from "../../common/v1/common_pb"; + +/** + * Describes the file opentelemetry/proto/trace/v1/trace.proto. + */ +export declare const file_opentelemetry_proto_trace_v1_trace: GenFile; + +/** + * TracesData represents the traces data that can be stored in a persistent storage, + * OR can be embedded by other protocols that transfer OTLP traces data but do + * not implement the OTLP protocol. + * + * The main difference between this message and collector protocol is that + * in this message there will not be any "control" or "metadata" specific to + * OTLP protocol. + * + * When new fields are added into this message, the OTLP request MUST be updated + * as well. + * + * @generated from message opentelemetry.proto.trace.v1.TracesData + */ +export declare type TracesData = Message<"opentelemetry.proto.trace.v1.TracesData"> & { + /** + * An array of ResourceSpans. + * For data coming from a single resource this array will typically contain + * one element. Intermediary nodes that receive data from multiple origins + * typically batch the data before forwarding further and in that case this + * array will contain multiple elements. + * + * @generated from field: repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1; + */ + resourceSpans: ResourceSpans[]; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.TracesData. + * Use `create(TracesDataSchema)` to create a new message. + */ +export declare const TracesDataSchema: GenMessage; + +/** + * A collection of ScopeSpans from a Resource. + * + * @generated from message opentelemetry.proto.trace.v1.ResourceSpans + */ +export declare type ResourceSpans = Message<"opentelemetry.proto.trace.v1.ResourceSpans"> & { + /** + * The resource for the spans in this message. + * If this field is not set then no resource info is known. + * + * @generated from field: opentelemetry.proto.resource.v1.Resource resource = 1; + */ + resource?: Resource | undefined; + + /** + * A list of ScopeSpans that originate from a resource. + * + * @generated from field: repeated opentelemetry.proto.trace.v1.ScopeSpans scope_spans = 2; + */ + scopeSpans: ScopeSpans[]; + + /** + * The Schema URL, if known. This is the identifier of the Schema that the resource data + * is recorded in. Notably, the last part of the URL path is the version number of the + * schema: http[s]://server[:port]/path/. To learn more about Schema URL see + * https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + * This schema_url applies to the data in the "resource" field. It does not apply + * to the data in the "scope_spans" field which have their own schema_url field. + * + * @generated from field: string schema_url = 3; + */ + schemaUrl: string; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.ResourceSpans. + * Use `create(ResourceSpansSchema)` to create a new message. + */ +export declare const ResourceSpansSchema: GenMessage; + +/** + * A collection of Spans produced by an InstrumentationScope. + * + * @generated from message opentelemetry.proto.trace.v1.ScopeSpans + */ +export declare type ScopeSpans = Message<"opentelemetry.proto.trace.v1.ScopeSpans"> & { + /** + * The instrumentation scope information for the spans in this message. + * Semantically when InstrumentationScope isn't set, it is equivalent with + * an empty instrumentation scope name (unknown). + * + * @generated from field: opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + */ + scope?: InstrumentationScope | undefined; + + /** + * A list of Spans that originate from an instrumentation scope. + * + * @generated from field: repeated opentelemetry.proto.trace.v1.Span spans = 2; + */ + spans: Span[]; + + /** + * The Schema URL, if known. This is the identifier of the Schema that the span data + * is recorded in. Notably, the last part of the URL path is the version number of the + * schema: http[s]://server[:port]/path/. To learn more about Schema URL see + * https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + * This schema_url applies to the data in the "scope" field and all spans and span + * events in the "spans" field. + * + * @generated from field: string schema_url = 3; + */ + schemaUrl: string; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.ScopeSpans. + * Use `create(ScopeSpansSchema)` to create a new message. + */ +export declare const ScopeSpansSchema: GenMessage; + +/** + * A Span represents a single operation performed by a single component of the system. + * + * The next available field id is 17. + * + * @generated from message opentelemetry.proto.trace.v1.Span + */ +export declare type Span = Message<"opentelemetry.proto.trace.v1.Span"> & { + /** + * A unique identifier for a trace. All spans from the same trace share + * the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + * of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + * is zero-length and thus is also invalid). + * + * This field is required. + * + * @generated from field: bytes trace_id = 1; + */ + traceId: Uint8Array; + + /** + * A unique identifier for a span within a trace, assigned when the span + * is created. The ID is an 8-byte array. An ID with all zeroes OR of length + * other than 8 bytes is considered invalid (empty string in OTLP/JSON + * is zero-length and thus is also invalid). + * + * This field is required. + * + * @generated from field: bytes span_id = 2; + */ + spanId: Uint8Array; + + /** + * trace_state conveys information about request position in multiple distributed tracing graphs. + * It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header + * See also https://github.com/w3c/distributed-tracing for more details about this field. + * + * @generated from field: string trace_state = 3; + */ + traceState: string; + + /** + * The `span_id` of this span's parent span. If this is a root span, then this + * field must be empty. The ID is an 8-byte array. + * + * @generated from field: bytes parent_span_id = 4; + */ + parentSpanId: Uint8Array; + + /** + * Flags, a bit field. + * + * Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + * Context specification. To read the 8-bit W3C trace flag, use + * `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + * + * See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + * + * Bits 8 and 9 represent the 3 states of whether a span's parent + * is remote. The states are (unknown, is not remote, is remote). + * To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + * To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + * + * When creating span messages, if the message is logically forwarded from another source + * with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + * be copied as-is. If creating from a source that does not have an equivalent flags field + * (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + * be set to zero. + * Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + * + * [Optional]. + * + * @generated from field: fixed32 flags = 16; + */ + flags: number; + + /** + * A description of the span's operation. + * + * For example, the name can be a qualified method name or a file name + * and a line number where the operation is called. A best practice is to use + * the same display name at the same call point in an application. + * This makes it easier to correlate spans in different traces. + * + * This field is semantically required to be set to non-empty string. + * Empty value is equivalent to an unknown span name. + * + * This field is required. + * + * @generated from field: string name = 5; + */ + name: string; + + /** + * Distinguishes between spans generated in a particular context. For example, + * two spans with the same name may be distinguished using `CLIENT` (caller) + * and `SERVER` (callee) to identify queueing latency associated with the span. + * + * @generated from field: opentelemetry.proto.trace.v1.Span.SpanKind kind = 6; + */ + kind: Span_SpanKind; + + /** + * The start time of the span. On the client side, this is the time + * kept by the local machine where the span execution starts. On the server side, this + * is the time when the server's application handler starts running. + * Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + * + * This field is semantically required and it is expected that end_time >= start_time. + * + * @generated from field: fixed64 start_time_unix_nano = 7; + */ + startTimeUnixNano: bigint; + + /** + * The end time of the span. On the client side, this is the time + * kept by the local machine where the span execution ends. On the server side, this + * is the time when the server application handler stops running. + * Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + * + * This field is semantically required and it is expected that end_time >= start_time. + * + * @generated from field: fixed64 end_time_unix_nano = 8; + */ + endTimeUnixNano: bigint; + + /** + * A collection of key/value pairs. Note, global attributes + * like server name can be set using the resource API. Examples of attributes: + * + * "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + * "/http/server_latency": 300 + * "example.com/myattribute": true + * "example.com/score": 10.239 + * + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + */ + attributes: KeyValue[]; + + /** + * The number of attributes that were discarded. Attributes + * can be discarded because their keys are too long or because there are too many + * attributes. If this value is 0, then no attributes were dropped. + * + * @generated from field: uint32 dropped_attributes_count = 10; + */ + droppedAttributesCount: number; + + /** + * A collection of Event items. + * + * @generated from field: repeated opentelemetry.proto.trace.v1.Span.Event events = 11; + */ + events: Span_Event[]; + + /** + * The number of dropped events. If the value is 0, then no + * events were dropped. + * + * @generated from field: uint32 dropped_events_count = 12; + */ + droppedEventsCount: number; + + /** + * A collection of Links, which are references from this span to a span + * in the same or different trace. + * + * @generated from field: repeated opentelemetry.proto.trace.v1.Span.Link links = 13; + */ + links: Span_Link[]; + + /** + * The number of dropped links after the maximum size was + * enforced. If this value is 0, then no links were dropped. + * + * @generated from field: uint32 dropped_links_count = 14; + */ + droppedLinksCount: number; + + /** + * An optional final status for this span. Semantically when Status isn't set, it means + * span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + * + * @generated from field: opentelemetry.proto.trace.v1.Status status = 15; + */ + status?: Status | undefined; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.Span. + * Use `create(SpanSchema)` to create a new message. + */ +export declare const SpanSchema: GenMessage; + +/** + * Event is a time-stamped annotation of the span, consisting of user-supplied + * text description and key-value pairs. + * + * @generated from message opentelemetry.proto.trace.v1.Span.Event + */ +export declare type Span_Event = Message<"opentelemetry.proto.trace.v1.Span.Event"> & { + /** + * The time the event occurred. + * + * @generated from field: fixed64 time_unix_nano = 1; + */ + timeUnixNano: bigint; + + /** + * The name of the event. + * This field is semantically required to be set to non-empty string. + * + * @generated from field: string name = 2; + */ + name: string; + + /** + * A collection of attribute key/value pairs on the event. + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + */ + attributes: KeyValue[]; + + /** + * The number of dropped attributes. If the value is 0, + * then no attributes were dropped. + * + * @generated from field: uint32 dropped_attributes_count = 4; + */ + droppedAttributesCount: number; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.Span.Event. + * Use `create(Span_EventSchema)` to create a new message. + */ +export declare const Span_EventSchema: GenMessage; + +/** + * A pointer from the current span to another span in the same trace or in a + * different trace. For example, this can be used in batching operations, + * where a single batch handler processes multiple requests from different + * traces or when the handler receives a request from a different project. + * + * @generated from message opentelemetry.proto.trace.v1.Span.Link + */ +export declare type Span_Link = Message<"opentelemetry.proto.trace.v1.Span.Link"> & { + /** + * A unique identifier of a trace that this linked span is part of. The ID is a + * 16-byte array. + * + * @generated from field: bytes trace_id = 1; + */ + traceId: Uint8Array; + + /** + * A unique identifier for the linked span. The ID is an 8-byte array. + * + * @generated from field: bytes span_id = 2; + */ + spanId: Uint8Array; + + /** + * The trace_state associated with the link. + * + * @generated from field: string trace_state = 3; + */ + traceState: string; + + /** + * A collection of attribute key/value pairs on the link. + * Attribute keys MUST be unique (it is not allowed to have more than one + * attribute with the same key). + * The behavior of software that receives duplicated keys can be unpredictable. + * + * @generated from field: repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + */ + attributes: KeyValue[]; + + /** + * The number of dropped attributes. If the value is 0, + * then no attributes were dropped. + * + * @generated from field: uint32 dropped_attributes_count = 5; + */ + droppedAttributesCount: number; + + /** + * Flags, a bit field. + * + * Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + * Context specification. To read the 8-bit W3C trace flag, use + * `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + * + * See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + * + * Bits 8 and 9 represent the 3 states of whether the link is remote. + * The states are (unknown, is not remote, is remote). + * To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + * To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + * + * Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + * When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + * + * [Optional]. + * + * @generated from field: fixed32 flags = 6; + */ + flags: number; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.Span.Link. + * Use `create(Span_LinkSchema)` to create a new message. + */ +export declare const Span_LinkSchema: GenMessage; + +/** + * SpanKind is the type of span. Can be used to specify additional relationships between spans + * in addition to a parent/child relationship. + * + * @generated from enum opentelemetry.proto.trace.v1.Span.SpanKind + */ +export enum Span_SpanKind { + /** + * Unspecified. Do NOT use as default. + * Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + * + * @generated from enum value: SPAN_KIND_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * Indicates that the span represents an internal operation within an application, + * as opposed to an operation happening at the boundaries. Default value. + * + * @generated from enum value: SPAN_KIND_INTERNAL = 1; + */ + INTERNAL = 1, + + /** + * Indicates that the span covers server-side handling of an RPC or other + * remote network request. + * + * @generated from enum value: SPAN_KIND_SERVER = 2; + */ + SERVER = 2, + + /** + * Indicates that the span describes a request to some remote service. + * + * @generated from enum value: SPAN_KIND_CLIENT = 3; + */ + CLIENT = 3, + + /** + * Indicates that the span describes a producer sending a message to a broker. + * Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + * between producer and consumer spans. A PRODUCER span ends when the message was accepted + * by the broker while the logical processing of the message might span a much longer time. + * + * @generated from enum value: SPAN_KIND_PRODUCER = 4; + */ + PRODUCER = 4, + + /** + * Indicates that the span describes consumer receiving a message from a broker. + * Like the PRODUCER kind, there is often no direct critical path latency relationship + * between producer and consumer spans. + * + * @generated from enum value: SPAN_KIND_CONSUMER = 5; + */ + CONSUMER = 5, +} + +/** + * Describes the enum opentelemetry.proto.trace.v1.Span.SpanKind. + */ +export declare const Span_SpanKindSchema: GenEnum; + +/** + * The Status type defines a logical error model that is suitable for different + * programming environments, including REST APIs and RPC APIs. + * + * @generated from message opentelemetry.proto.trace.v1.Status + */ +export declare type Status = Message<"opentelemetry.proto.trace.v1.Status"> & { + /** + * A developer-facing human readable error message. + * + * @generated from field: string message = 2; + */ + message: string; + + /** + * The status code. + * + * @generated from field: opentelemetry.proto.trace.v1.Status.StatusCode code = 3; + */ + code: Status_StatusCode; +}; + +/** + * Describes the message opentelemetry.proto.trace.v1.Status. + * Use `create(StatusSchema)` to create a new message. + */ +export declare const StatusSchema: GenMessage; + +/** + * For the semantics of status codes see + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + * + * @generated from enum opentelemetry.proto.trace.v1.Status.StatusCode + */ +export enum Status_StatusCode { + /** + * The default status. + * + * @generated from enum value: STATUS_CODE_UNSET = 0; + */ + UNSET = 0, + + /** + * The Span has been validated by an Application developer or Operator to + * have completed successfully. + * + * @generated from enum value: STATUS_CODE_OK = 1; + */ + OK = 1, + + /** + * The Span contains an error. + * + * @generated from enum value: STATUS_CODE_ERROR = 2; + */ + ERROR = 2, +} + +/** + * Describes the enum opentelemetry.proto.trace.v1.Status.StatusCode. + */ +export declare const Status_StatusCodeSchema: GenEnum; + +/** + * SpanFlags represents constants used to interpret the + * Span.flags field, which is protobuf 'fixed32' type and is to + * be used as bit-fields. Each non-zero value defined in this enum is + * a bit-mask. To extract the bit-field, for example, use an + * expression like: + * + * (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) + * + * See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + * + * Note that Span flags were introduced in version 1.1 of the + * OpenTelemetry protocol. Older Span producers do not set this + * field, consequently consumers should not rely on the absence of a + * particular flag bit to indicate the presence of a particular feature. + * + * @generated from enum opentelemetry.proto.trace.v1.SpanFlags + */ +export enum SpanFlags { + /** + * The zero value for the enum. Should not be used for comparisons. + * Instead use bitwise "and" with the appropriate mask as shown above. + * + * @generated from enum value: SPAN_FLAGS_DO_NOT_USE = 0; + */ + DO_NOT_USE = 0, + + /** + * Bits 0-7 are used for trace flags. + * + * @generated from enum value: SPAN_FLAGS_TRACE_FLAGS_MASK = 255; + */ + TRACE_FLAGS_MASK = 255, + + /** + * Bits 8 and 9 are used to indicate that the parent span or link span is remote. + * Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + * Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + * + * @generated from enum value: SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 256; + */ + CONTEXT_HAS_IS_REMOTE_MASK = 256, + + /** + * @generated from enum value: SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 512; + */ + CONTEXT_IS_REMOTE_MASK = 512, +} + +/** + * Describes the enum opentelemetry.proto.trace.v1.SpanFlags. + */ +export declare const SpanFlagsSchema: GenEnum; + diff --git a/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.js b/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.js new file mode 100644 index 00000000..1d18428b --- /dev/null +++ b/ui/src/generated/opentelemetry/proto/trace/v1/trace_pb.js @@ -0,0 +1,134 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// @generated by protoc-gen-es v2.13.0 +// @generated from file opentelemetry/proto/trace/v1/trace.proto (package opentelemetry.proto.trace.v1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; +import { file_opentelemetry_proto_common_v1_common } from "../../common/v1/common_pb"; +import { file_opentelemetry_proto_resource_v1_resource } from "../../resource/v1/resource_pb"; + +/** + * Describes the file opentelemetry/proto/trace/v1/trace.proto. + */ +export const file_opentelemetry_proto_trace_v1_trace = /*@__PURE__*/ + fileDesc("CihvcGVudGVsZW1ldHJ5L3Byb3RvL3RyYWNlL3YxL3RyYWNlLnByb3RvEhxvcGVudGVsZW1ldHJ5LnByb3RvLnRyYWNlLnYxIlEKClRyYWNlc0RhdGESQwoOcmVzb3VyY2Vfc3BhbnMYASADKAsyKy5vcGVudGVsZW1ldHJ5LnByb3RvLnRyYWNlLnYxLlJlc291cmNlU3BhbnMipwEKDVJlc291cmNlU3BhbnMSOwoIcmVzb3VyY2UYASABKAsyKS5vcGVudGVsZW1ldHJ5LnByb3RvLnJlc291cmNlLnYxLlJlc291cmNlEj0KC3Njb3BlX3NwYW5zGAIgAygLMigub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TY29wZVNwYW5zEhIKCnNjaGVtYV91cmwYAyABKAlKBgjoBxDpByKXAQoKU2NvcGVTcGFucxJCCgVzY29wZRgBIAEoCzIzLm9wZW50ZWxlbWV0cnkucHJvdG8uY29tbW9uLnYxLkluc3RydW1lbnRhdGlvblNjb3BlEjEKBXNwYW5zGAIgAygLMiIub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TcGFuEhIKCnNjaGVtYV91cmwYAyABKAkihAgKBFNwYW4SEAoIdHJhY2VfaWQYASABKAwSDwoHc3Bhbl9pZBgCIAEoDBITCgt0cmFjZV9zdGF0ZRgDIAEoCRIWCg5wYXJlbnRfc3Bhbl9pZBgEIAEoDBINCgVmbGFncxgQIAEoBxIMCgRuYW1lGAUgASgJEjkKBGtpbmQYBiABKA4yKy5vcGVudGVsZW1ldHJ5LnByb3RvLnRyYWNlLnYxLlNwYW4uU3BhbktpbmQSHAoUc3RhcnRfdGltZV91bml4X25hbm8YByABKAYSGgoSZW5kX3RpbWVfdW5peF9uYW5vGAggASgGEjsKCmF0dHJpYnV0ZXMYCSADKAsyJy5vcGVudGVsZW1ldHJ5LnByb3RvLmNvbW1vbi52MS5LZXlWYWx1ZRIgChhkcm9wcGVkX2F0dHJpYnV0ZXNfY291bnQYCiABKA0SOAoGZXZlbnRzGAsgAygLMigub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TcGFuLkV2ZW50EhwKFGRyb3BwZWRfZXZlbnRzX2NvdW50GAwgASgNEjYKBWxpbmtzGA0gAygLMicub3BlbnRlbGVtZXRyeS5wcm90by50cmFjZS52MS5TcGFuLkxpbmsSGwoTZHJvcHBlZF9saW5rc19jb3VudBgOIAEoDRI0CgZzdGF0dXMYDyABKAsyJC5vcGVudGVsZW1ldHJ5LnByb3RvLnRyYWNlLnYxLlN0YXR1cxqMAQoFRXZlbnQSFgoOdGltZV91bml4X25hbm8YASABKAYSDAoEbmFtZRgCIAEoCRI7CgphdHRyaWJ1dGVzGAMgAygLMicub3BlbnRlbGVtZXRyeS5wcm90by5jb21tb24udjEuS2V5VmFsdWUSIAoYZHJvcHBlZF9hdHRyaWJ1dGVzX2NvdW50GAQgASgNGqwBCgRMaW5rEhAKCHRyYWNlX2lkGAEgASgMEg8KB3NwYW5faWQYAiABKAwSEwoLdHJhY2Vfc3RhdGUYAyABKAkSOwoKYXR0cmlidXRlcxgEIAMoCzInLm9wZW50ZWxlbWV0cnkucHJvdG8uY29tbW9uLnYxLktleVZhbHVlEiAKGGRyb3BwZWRfYXR0cmlidXRlc19jb3VudBgFIAEoDRINCgVmbGFncxgGIAEoByKZAQoIU3BhbktpbmQSGQoVU1BBTl9LSU5EX1VOU1BFQ0lGSUVEEAASFgoSU1BBTl9LSU5EX0lOVEVSTkFMEAESFAoQU1BBTl9LSU5EX1NFUlZFUhACEhQKEFNQQU5fS0lORF9DTElFTlQQAxIWChJTUEFOX0tJTkRfUFJPRFVDRVIQBBIWChJTUEFOX0tJTkRfQ09OU1VNRVIQBSKuAQoGU3RhdHVzEg8KB21lc3NhZ2UYAiABKAkSPQoEY29kZRgDIAEoDjIvLm9wZW50ZWxlbWV0cnkucHJvdG8udHJhY2UudjEuU3RhdHVzLlN0YXR1c0NvZGUiTgoKU3RhdHVzQ29kZRIVChFTVEFUVVNfQ09ERV9VTlNFVBAAEhIKDlNUQVRVU19DT0RFX09LEAESFQoRU1RBVFVTX0NPREVfRVJST1IQAkoECAEQAiqcAQoJU3BhbkZsYWdzEhkKFVNQQU5fRkxBR1NfRE9fTk9UX1VTRRAAEiAKG1NQQU5fRkxBR1NfVFJBQ0VfRkxBR1NfTUFTSxD/ARIqCiVTUEFOX0ZMQUdTX0NPTlRFWFRfSEFTX0lTX1JFTU9URV9NQVNLEIACEiYKIVNQQU5fRkxBR1NfQ09OVEVYVF9JU19SRU1PVEVfTUFTSxCABEJ3Ch9pby5vcGVudGVsZW1ldHJ5LnByb3RvLnRyYWNlLnYxQgpUcmFjZVByb3RvUAFaJ2dvLm9wZW50ZWxlbWV0cnkuaW8vcHJvdG8vb3RscC90cmFjZS92MaoCHE9wZW5UZWxlbWV0cnkuUHJvdG8uVHJhY2UuVjFiBnByb3RvMw", [file_opentelemetry_proto_common_v1_common, file_opentelemetry_proto_resource_v1_resource]); + +/** + * Describes the message opentelemetry.proto.trace.v1.TracesData. + * Use `create(TracesDataSchema)` to create a new message. + */ +export const TracesDataSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 0); + +/** + * Describes the message opentelemetry.proto.trace.v1.ResourceSpans. + * Use `create(ResourceSpansSchema)` to create a new message. + */ +export const ResourceSpansSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 1); + +/** + * Describes the message opentelemetry.proto.trace.v1.ScopeSpans. + * Use `create(ScopeSpansSchema)` to create a new message. + */ +export const ScopeSpansSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 2); + +/** + * Describes the message opentelemetry.proto.trace.v1.Span. + * Use `create(SpanSchema)` to create a new message. + */ +export const SpanSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 3); + +/** + * Describes the message opentelemetry.proto.trace.v1.Span.Event. + * Use `create(Span_EventSchema)` to create a new message. + */ +export const Span_EventSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 3, 0); + +/** + * Describes the message opentelemetry.proto.trace.v1.Span.Link. + * Use `create(Span_LinkSchema)` to create a new message. + */ +export const Span_LinkSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 3, 1); + +/** + * Describes the enum opentelemetry.proto.trace.v1.Span.SpanKind. + */ +export const Span_SpanKindSchema = /*@__PURE__*/ + enumDesc(file_opentelemetry_proto_trace_v1_trace, 3, 0); + +/** + * SpanKind is the type of span. Can be used to specify additional relationships between spans + * in addition to a parent/child relationship. + * + * @generated from enum opentelemetry.proto.trace.v1.Span.SpanKind + */ +export const Span_SpanKind = /*@__PURE__*/ + tsEnum(Span_SpanKindSchema); + +/** + * Describes the message opentelemetry.proto.trace.v1.Status. + * Use `create(StatusSchema)` to create a new message. + */ +export const StatusSchema = /*@__PURE__*/ + messageDesc(file_opentelemetry_proto_trace_v1_trace, 4); + +/** + * Describes the enum opentelemetry.proto.trace.v1.Status.StatusCode. + */ +export const Status_StatusCodeSchema = /*@__PURE__*/ + enumDesc(file_opentelemetry_proto_trace_v1_trace, 4, 0); + +/** + * For the semantics of status codes see + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + * + * @generated from enum opentelemetry.proto.trace.v1.Status.StatusCode + */ +export const Status_StatusCode = /*@__PURE__*/ + tsEnum(Status_StatusCodeSchema); + +/** + * Describes the enum opentelemetry.proto.trace.v1.SpanFlags. + */ +export const SpanFlagsSchema = /*@__PURE__*/ + enumDesc(file_opentelemetry_proto_trace_v1_trace, 0); + +/** + * SpanFlags represents constants used to interpret the + * Span.flags field, which is protobuf 'fixed32' type and is to + * be used as bit-fields. Each non-zero value defined in this enum is + * a bit-mask. To extract the bit-field, for example, use an + * expression like: + * + * (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) + * + * See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + * + * Note that Span flags were introduced in version 1.1 of the + * OpenTelemetry protocol. Older Span producers do not set this + * field, consequently consumers should not rely on the absence of a + * particular flag bit to indicate the presence of a particular feature. + * + * @generated from enum opentelemetry.proto.trace.v1.SpanFlags + */ +export const SpanFlags = /*@__PURE__*/ + tsEnum(SpanFlagsSchema); + diff --git a/ui/src/lib/console-capture.ts b/ui/src/lib/console-capture.ts index 6809ed63..369fe337 100644 --- a/ui/src/lib/console-capture.ts +++ b/ui/src/lib/console-capture.ts @@ -19,9 +19,9 @@ export function installConsoleCapture(): void { function capture( level: ConsoleEntry['level'], - original: (...args: any[]) => void, + original: (...args: unknown[]) => void, ) { - return (...args: any[]) => { + return (...args: unknown[]) => { original.apply(console, args); const entry: ConsoleEntry = { timestamp: new Date().toISOString(), diff --git a/ui/src/lib/liveSessions.ts b/ui/src/lib/liveSessions.ts new file mode 100644 index 00000000..c195926c --- /dev/null +++ b/ui/src/lib/liveSessions.ts @@ -0,0 +1,77 @@ +import { fromJson, toJson } from '@bufbuild/protobuf'; +import { timestampMs, ValueSchema } from '@bufbuild/protobuf/wkt'; +import type { Value } from '@bufbuild/protobuf/wkt'; +import { WatchSessionsResponseSchema } from '../generated/agentevals/api/v1alpha1/sessions_pb'; +import type { WatchSessionsResponse } from '../generated/agentevals/api/v1alpha1/sessions_pb'; +import { SessionState } from '../generated/agentevals/api/v1alpha1/views_pb'; +import type { Session, TurnView, Usage } from '../generated/agentevals/api/v1alpha1/views_pb'; +import { Status_StatusCode } from '../generated/opentelemetry/proto/trace/v1/trace_pb'; + +export type { Session, TurnView } from '../generated/agentevals/api/v1alpha1/views_pb'; + +/** One frame of ``/stream/ui-updates``: a protojson ``WatchSessionsResponse``. */ +export function decodeFrame(data: string): WatchSessionsResponse { + return fromJson(WatchSessionsResponseSchema, JSON.parse(data), { ignoreUnknownFields: true }); +} + +export function sessionId(session: Session): string { + return session.summary?.id ?? ''; +} + +export function isComplete(session: Session): boolean { + return session.summary?.state === SessionState.COMPLETE; +} + +export function startedAtMs(session: Session): number { + const startedAt = session.summary?.startedAt; + return startedAt ? timestampMs(startedAt) : 0; +} + +export function startedAtIso(session: Session): string | undefined { + const ms = startedAtMs(session); + return ms ? new Date(ms).toISOString() : undefined; +} + +/** The session's earliest trace, the id evaluation results are keyed by. */ +export function primaryTraceId(session: Session): string { + return session.summary?.traceIds[0] ?? ''; +} + +export function isFailed(turn: TurnView): boolean { + return turn.status === Status_StatusCode.ERROR; +} + +/** Token counts as numbers; ``undefined`` where the telemetry recorded none. */ +export function tokens(usage: Usage | undefined) { + const n = (v: bigint | undefined) => (v === undefined ? undefined : Number(v)); + return { + input: n(usage?.inputTokens), + output: n(usage?.outputTokens), + cacheRead: n(usage?.cacheReadInputTokens), + cacheWrite: n(usage?.cacheWriteInputTokens), + }; +} + +export function totalTokens(usage: Usage | undefined): number { + const t = tokens(usage); + return (t.input ?? 0) + (t.output ?? 0); +} + +/** A tool argument or result as plain JSON, or ``undefined`` when not recorded. */ +export function jsonValue(value: Value | undefined): unknown { + return value === undefined ? undefined : toJson(ValueSchema, value); +} + +/** Every gap reason of the session and its turns, once each. */ +export function gapReasons(session: Session): string[] { + const gaps = [...(session.summary?.gaps ?? []), ...session.turns.flatMap(t => t.gaps)]; + return Array.from(new Set(gaps.map(g => g.reason))); +} + +export function firstProvider(session: Session): string | undefined { + for (const turn of session.turns) { + const provider = turn.modelCalls.find(c => c.provider)?.provider; + if (provider) return provider; + } + return undefined; +} diff --git a/ui/src/lib/toolPairing.ts b/ui/src/lib/toolPairing.ts new file mode 100644 index 00000000..0ee4611a --- /dev/null +++ b/ui/src/lib/toolPairing.ts @@ -0,0 +1,22 @@ +interface Named { + name: string; + id?: string | null; +} + +/** + * The response for each call, or undefined. Ids decide when both sides have one; otherwise the + * next unused response with the same name, so repeated calls to one tool keep their own results. + */ +export function pairToolResponses(calls: C[], responses: R[]): Array { + const used = new Set(); + const take = (match: (r: R) => boolean) => { + const i = responses.findIndex((r, j) => !used.has(j) && match(r)); + if (i < 0) return undefined; + used.add(i); + return responses[i]; + }; + return calls.map(tc => + (tc.id ? take(r => r.id === tc.id) : undefined) + ?? take(r => r.name === tc.name && !(tc.id && r.id)), + ); +} diff --git a/ui/src/lib/trace-loader.ts b/ui/src/lib/trace-loader.ts index f0c7d388..de69b170 100644 --- a/ui/src/lib/trace-loader.ts +++ b/ui/src/lib/trace-loader.ts @@ -1,9 +1,10 @@ +import type { JsonObject, JsonValue } from '@bufbuild/protobuf'; import type { Trace, Span, Log } from './types'; interface JaegerTag { key: string; type: string; - value: any; + value: JsonValue; } interface JaegerLog { @@ -47,6 +48,42 @@ interface OtlpAttribute { }; } +interface OtlpSpan { + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + startTimeUnixNano?: string | number; + endTimeUnixNano?: string | number; + attributes?: OtlpAttribute[]; +} + +interface OtlpScope { + name?: string; + version?: string; +} + +interface OtlpScopeSpans { + scope?: OtlpScope; + instrumentationLibrary?: OtlpScope; + spans?: OtlpSpan[]; +} + +interface OtlpResourceSpans { + resource?: { attributes?: OtlpAttribute[] }; + scopeSpans?: OtlpScopeSpans[]; + instrumentationLibrarySpans?: OtlpScopeSpans[]; +} + +interface OtlpDocument { + resourceSpans?: OtlpResourceSpans[]; + batches?: OtlpResourceSpans[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + // OTLP/JSON encodes int64 values as strings to preserve precision past // Number.MAX_SAFE_INTEGER (2^53 - 1). Convert only when the value still // fits a JavaScript safe integer; otherwise keep the original string so @@ -58,8 +95,8 @@ function parseOtlpIntValue(value: number | string): number | string { return Number.isSafeInteger(parsed) ? parsed : value; } -function extractOtlpAttributes(attrs: OtlpAttribute[] | undefined): Record { - const tags: Record = {}; +function extractOtlpAttributes(attrs: OtlpAttribute[] | undefined): JsonObject { + const tags: JsonObject = {}; for (const attr of attrs || []) { const v = attr.value; if (!v) continue; @@ -71,11 +108,11 @@ function extractOtlpAttributes(attrs: OtlpAttribute[] | undefined): Record = {}): Span { +function buildOtlpSpan(otlpSpan: OtlpSpan, extraTags: JsonObject = {}): Span { const tags = { ...extraTags, ...extractOtlpAttributes(otlpSpan.attributes) }; - const startTimeNs = parseInt(otlpSpan.startTimeUnixNano || '0'); - const endTimeNs = parseInt(otlpSpan.endTimeUnixNano || '0'); + const startTimeNs = parseInt(String(otlpSpan.startTimeUnixNano || '0')); + const endTimeNs = parseInt(String(otlpSpan.endTimeUnixNano || '0')); const startTimeUs = Math.floor(startTimeNs / 1000); const durationUs = Math.floor((endTimeNs - startTimeNs) / 1000); @@ -141,7 +178,7 @@ function loadOtlpJsonlTraces(fileContent: string): Trace[] { const spans: Span[] = []; for (const line of lines) { if (!line.trim()) continue; - spans.push(buildOtlpSpan(JSON.parse(line))); + spans.push(buildOtlpSpan(JSON.parse(line) as OtlpSpan)); } return buildTracesFromSpans(spans); } @@ -151,16 +188,16 @@ function loadOtlpJsonlTraces(fileContent: string): Trace[] { * batches, or Tempo v2 ``trace`` wrapper). The object should already be * unwrapped from any ``trace`` envelope. */ -function loadOtlpDocTraces(doc: any): Trace[] { - const resourceSpans: any[] = doc.resourceSpans || doc.batches || []; +function loadOtlpDocTraces(doc: OtlpDocument): Trace[] { + const resourceSpans = doc.resourceSpans || doc.batches || []; const spans: Span[] = []; for (const rs of resourceSpans) { const resourceTags = extractOtlpAttributes(rs.resource?.attributes); - const scopeSpans: any[] = rs.scopeSpans || rs.instrumentationLibrarySpans || []; + const scopeSpans = rs.scopeSpans || rs.instrumentationLibrarySpans || []; for (const ss of scopeSpans) { const scope = ss.scope || ss.instrumentationLibrary || {}; - const scopeTags: Record = { ...resourceTags }; + const scopeTags: JsonObject = { ...resourceTags }; if (scope.name) scopeTags['otel.scope.name'] = scope.name; if (scope.version) scopeTags['otel.scope.version'] = scope.version; for (const otlpSpan of ss.spans || []) { @@ -229,7 +266,7 @@ export async function loadTraces(fileContent: string): Promise { const trimmed = fileContent.trim(); if (!trimmed) return []; - let singleDoc: any = null; + let singleDoc: unknown; try { singleDoc = JSON.parse(trimmed); } catch { @@ -239,18 +276,17 @@ export async function loadTraces(fileContent: string): Promise { throw new Error('Invalid trace file: not valid JSON or JSONL.'); } - if (singleDoc && typeof singleDoc === 'object' && !Array.isArray(singleDoc)) { + if (isRecord(singleDoc)) { let doc = singleDoc; - if (doc.trace && typeof doc.trace === 'object' && - ('resourceSpans' in doc.trace || 'batches' in doc.trace)) { + if (isRecord(doc.trace) && ('resourceSpans' in doc.trace || 'batches' in doc.trace)) { doc = doc.trace; } if ('resourceSpans' in doc || 'batches' in doc) { - return loadOtlpDocTraces(doc); + return loadOtlpDocTraces(doc as OtlpDocument); } if (Array.isArray(doc.data)) { - return parseJaegerData(doc as JaegerData); + return parseJaegerData(doc as unknown as JaegerData); } } @@ -274,8 +310,8 @@ function extractParentSpanId(references?: JaegerReference[]): string | null { /** * Flatten Jaeger tags array to a key-value object */ -function flattenTags(tags: JaegerTag[]): Record { - const result: Record = {}; +function flattenTags(tags: JaegerTag[]): JsonObject { + const result: JsonObject = {}; for (const tag of tags) { result[tag.key] = tag.value; @@ -293,28 +329,3 @@ function flattenLogs(logs: JaegerLog[]): Log[] { fields: flattenTags(log.fields), })); } - -/** - * Find spans by operation name - */ -export function findSpansByOperation(trace: Trace, operationName: string): Span[] { - return trace.allSpans.filter((span) => span.operationName.includes(operationName)); -} - -/** - * Find spans by tag key-value - */ -export function findSpansByTag(trace: Trace, tagKey: string, tagValue?: any): Span[] { - return trace.allSpans.filter((span) => { - if (!(tagKey in span.tags)) return false; - if (tagValue === undefined) return true; - return span.tags[tagKey] === tagValue; - }); -} - -/** - * Get span by ID - */ -export function getSpanById(trace: Trace, spanId: string): Span | undefined { - return trace.allSpans.find((span) => span.spanId === spanId); -} diff --git a/ui/src/lib/trace-patcher.ts b/ui/src/lib/trace-patcher.ts index 8868773c..5805ead8 100644 --- a/ui/src/lib/trace-patcher.ts +++ b/ui/src/lib/trace-patcher.ts @@ -228,7 +228,8 @@ function jaegerStore(rawSpan: RawSpan): AttrStore { function patchSpan(parsedFile: ParsedTraceFile, spanId: string, field: Field, newText: string): void { const locRef = parsedFile.spanIndex.get(spanId); if (!locRef) return; - const store = parsedFile.format === 'jaeger' ? jaegerStore(locRef.raw) : otlpStore(locRef.raw); + const raw = locRef.raw as RawSpan; + const store = parsedFile.format === 'jaeger' ? jaegerStore(raw) : otlpStore(raw); for (const key of MESSAGE_KEYS[field]) { const value = store.get(key); @@ -262,7 +263,21 @@ function patchIndexed(store: AttrStore, field: Field, newText: string): void { if (target !== null) store.set(`${prefix}${target}.content`, newText); } -function patchJson(jsonStr: string, patch: (data: any) => string): string | null { +type JsonRecord = Record; + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function textParts(parts: unknown): JsonRecord[] { + return Array.isArray(parts) ? parts.filter((p): p is JsonRecord => isRecord(p) && p.text !== undefined) : []; +} + +function roleOf(msg: unknown): string { + return isRecord(msg) && typeof msg.role === 'string' ? msg.role : ''; +} + +function patchJson(jsonStr: string, patch: (data: unknown) => string): string | null { try { return patch(JSON.parse(jsonStr)); } catch { @@ -270,43 +285,43 @@ function patchJson(jsonStr: string, patch: (data: any) => string): string | null } } -function patchAdkJsonValue(data: any, field: 'user' | 'response', newText: string): string { +function patchAdkJsonValue(data: unknown, field: 'user' | 'response', newText: string): string { + if (!isRecord(data)) return JSON.stringify(data); if (field === 'user') { const contents = data.contents; if (Array.isArray(contents)) { for (let i = contents.length - 1; i >= 0; i--) { - if (contents[i].role === 'user') { - const textParts = contents[i].parts?.filter((p: any) => p.text !== undefined); - if (textParts && textParts.length > 0) { - textParts[0].text = newText; + const content = contents[i]; + if (isRecord(content) && content.role === 'user') { + const parts = textParts(content.parts); + if (parts.length > 0) { + parts[0].text = newText; break; } } } } } else { - const parts = data.content?.parts; - if (Array.isArray(parts)) { - const textParts = parts.filter((p: any) => p.text !== undefined); - if (textParts.length > 0) { - textParts[0].text = newText; - } + const parts = isRecord(data.content) ? textParts(data.content.parts) : []; + if (parts.length > 0) { + parts[0].text = newText; } } return JSON.stringify(data); } -function hasText(msg: any): boolean { +function hasText(msg: unknown): boolean { + if (!isRecord(msg)) return false; if (typeof msg.content === 'string' && msg.content) return true; - if (Array.isArray(msg.content) && msg.content.some((item: any) => typeof item === 'object' && item.text)) return true; - return Array.isArray(msg.parts) && msg.parts.some((p: any) => typeof p === 'object' && p.type === 'text'); + if (Array.isArray(msg.content) && msg.content.some(item => isRecord(item) && item.text)) return true; + return Array.isArray(msg.parts) && msg.parts.some(p => isRecord(p) && p.type === 'text'); } -function patchGenAIJsonValue(data: any, field: 'user' | 'response', newText: string): string { +function patchGenAIJsonValue(data: unknown, field: 'user' | 'response', newText: string): string { if (!Array.isArray(data)) return JSON.stringify(data); // A tool request has no text to replace; leave it rather than invent an answer. - if (field === 'response' && !data.some((msg: any) => ASSISTANT_ROLES.includes(msg.role) && hasText(msg))) { + if (field === 'response' && !data.some(msg => ASSISTANT_ROLES.includes(roleOf(msg)) && hasText(msg))) { return JSON.stringify(data); } @@ -314,22 +329,22 @@ function patchGenAIJsonValue(data: any, field: 'user' | 'response', newText: str for (let i = data.length - 1; i >= 0; i--) { const msg = data[i]; - if (!targetRoles.includes(msg.role)) continue; + if (!isRecord(msg) || !targetRoles.includes(roleOf(msg))) continue; if (typeof msg.content === 'string') { msg.content = newText; break; } if (Array.isArray(msg.content)) { - const textItem = msg.content.find((item: any) => typeof item === 'object' && item.text); - if (textItem) { + const textItem = msg.content.find(item => isRecord(item) && item.text); + if (isRecord(textItem)) { textItem.text = newText; break; } } if (Array.isArray(msg.parts)) { - const textPart = msg.parts.find((p: any) => typeof p === 'object' && p.type === 'text'); - if (textPart) { + const textPart = msg.parts.find(p => isRecord(p) && p.type === 'text'); + if (isRecord(textPart)) { textPart.content = newText; break; } @@ -343,7 +358,7 @@ function patchGenAIJsonValue(data: any, field: 'user' | 'response', newText: str function serialize(parsedFile: ParsedTraceFile): string { if (parsedFile.format === 'otlp-jsonl') { - return parsedFile.rawData.map((line: any) => JSON.stringify(line)).join('\n'); + return (parsedFile.rawData as unknown[]).map(line => JSON.stringify(line)).join('\n'); } if (parsedFile.format === 'otlp-json') { return JSON.stringify(parsedFile.rawData); diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index 843031fe..a058403f 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -1,3 +1,6 @@ +import type { JsonObject } from '@bufbuild/protobuf'; +import type { Session } from './liveSessions'; + // API envelope export interface StandardResponse { data: T; @@ -7,7 +10,7 @@ export interface StandardResponse { // Core trace data structures export interface Log { timestamp: number; - fields: Record; + fields: JsonObject; } export interface Span { @@ -17,7 +20,7 @@ export interface Span { operationName: string; startTime: number; // microseconds duration: number; // microseconds - tags: Record; + tags: JsonObject; logs: Log[]; children: Span[]; } @@ -42,25 +45,25 @@ export interface Part { export interface FunctionCall { name: string; - args: Record; + args: JsonObject; id?: string; } export interface FunctionResponse { name: string; - response: Record; + response: JsonObject; id?: string; } export interface ToolCall { name: string; - args: Record; + args: JsonObject; id?: string; } export interface ToolResponse { name: string; - response: Record; + response: JsonObject; id?: string; } @@ -105,7 +108,7 @@ export type EvalStatus = 'PASSED' | 'FAILED' | 'NOT_EVALUATED' | 'ERROR'; export interface ToolCallComparison { name: string; - args: Record; + args: JsonObject; } export interface TrajectoryComparison { @@ -476,57 +479,6 @@ export const AVAILABLE_METRICS: MetricMetadata[] = [ } ]; -// Streaming / Live session types -export interface ConversationElement { - type: 'user_input' | 'tool_call' | 'tool_result' | 'agent_response' | 'turn_error'; - timestamp: number; - invocationId: string; - data: any; -} - -export interface ExternalEvaluation { - name: string; - scoreValue?: number | null; - scoreLabel?: string | null; - explanation?: string | null; - spanId: string; - sourceService?: string | null; -} - -export interface StreamingInvocation { - /** Span id of the turn's anchor span. */ - invocationId: string; - traceId?: string; - userText: string; - agentText: string; - toolCalls: Array<{ name: string; args: any; id?: string }>; - toolResponses?: Array<{ name: string; response: any; id?: string }>; - modelInfo?: Record; - status?: 'unset' | 'ok' | 'error'; - errorType?: string | null; - contentCaptured?: boolean; - warnings?: string[]; - /** ``gen_ai.evaluation.result`` events found in the telemetry (from other evaluators). */ - externalEvaluations?: ExternalEvaluation[]; -} - -export interface LiveSession { - sessionId: string; - traceId: string; - evalSetId: string | null; - spans: any[]; - status: 'active' | 'complete'; - metadata: Record; - invocations?: StreamingInvocation[]; - liveElements: ConversationElement[]; - liveStats: { - totalInputTokens: number; - totalOutputTokens: number; - model?: string; - }; - startedAt: string; -} - // Inspector-specific types export type ExtractionType = 'user_input' | 'tool_use' | 'tool_response' | 'final_response'; @@ -560,13 +512,14 @@ export type TraceFileFormat = 'jaeger' | 'otlp-json' | 'otlp-jsonl'; export interface SpanLocationRef { /** The span object inside the parsed file; patches edit it in place. */ - raw: any; + raw: object; } export interface ParsedTraceFile { format: TraceFileFormat; fileName: string; - rawData: any; + /** The parsed document; one entry per line for ``otlp-jsonl``. */ + rawData: unknown; spanIndex: Map; } @@ -594,10 +547,8 @@ export interface AnnotationQueueItem { model?: string; totalTokens?: number; annotation?: Annotation; - invocations?: StreamingInvocation[]; - liveElements?: ConversationElement[]; - liveStats?: { totalInputTokens: number; totalOutputTokens: number; model?: string }; - metadata?: Record; + /** The live session as it was when queued. */ + session: Session; } export interface AnnotationQueue { diff --git a/ui/src/lib/utils.ts b/ui/src/lib/utils.ts index 822c76dd..adc069e5 100644 --- a/ui/src/lib/utils.ts +++ b/ui/src/lib/utils.ts @@ -48,7 +48,7 @@ export function truncateTraceId(traceId: string, length: number = 12): string { /** * Get color for status */ -export function getStatusColor(status: EvalStatus): string { +export function getStatusColor(status: EvalStatus | 'PARTIAL'): string { switch (status) { case 'PASSED': return 'var(--status-success)'; @@ -66,7 +66,7 @@ export function getStatusColor(status: EvalStatus): string { /** * Get glow effect for status */ -export function getStatusGlow(status: EvalStatus): string { +export function getStatusGlow(status: EvalStatus | 'PARTIAL'): string { switch (status) { case 'PASSED': return 'var(--glow-success)'; @@ -190,11 +190,11 @@ function convertKeys(obj: unknown, rename: (key: string) => string): unknown { return obj; } -export function convertSnakeToCamel(obj: any): any { +export function convertSnakeToCamel(obj: unknown): unknown { return convertKeys(obj, (key) => key.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase())); } -export function convertCamelToSnake(obj: any): any { +export function convertCamelToSnake(obj: unknown): unknown { return convertKeys(obj, (key) => key.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`)); } diff --git a/uv.lock b/uv.lock index 7124698f..34f35276 100644 --- a/uv.lock +++ b/uv.lock @@ -39,14 +39,19 @@ dependencies = [ { name = "google-adk", extra = ["eval"] }, { name = "googleapis-common-protos" }, { name = "grpcio" }, + { name = "grpcio-health-checking" }, + { name = "grpcio-reflection" }, + { name = "grpcio-status" }, { name = "httpx" }, { name = "idna" }, { name = "litellm" }, { name = "nltk" }, { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-grpc" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, + { name = "protobuf" }, { name = "pyasn1" }, { name = "pygments" }, { name = "python-dotenv" }, @@ -101,7 +106,10 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.115.0" }, { name = "google-adk", extras = ["eval"], specifier = ">=2.5.0,<3" }, { name = "googleapis-common-protos", specifier = ">=1.72.0" }, - { name = "grpcio", specifier = ">=1.66" }, + { name = "grpcio", specifier = ">=1.78" }, + { name = "grpcio-health-checking", specifier = ">=1.78" }, + { name = "grpcio-reflection", specifier = ">=1.78" }, + { name = "grpcio-status", specifier = ">=1.78" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "httpx", marker = "extra == 'live'", specifier = ">=0.27.0" }, { name = "idna", specifier = ">=3.18" }, @@ -111,8 +119,10 @@ requires-dist = [ { name = "nltk", specifier = ">=3.10.2" }, { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.38" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.38" }, + { name = "opentelemetry-instrumentation-grpc", specifier = ">=0.59b0" }, { name = "opentelemetry-proto", specifier = ">=1.36.0" }, { name = "opentelemetry-sdk", specifier = ">=1.38" }, + { name = "protobuf", specifier = ">=6.33.5,<7" }, { name = "pyasn1", specifier = ">=0.6.4" }, { name = "pydantic-settings", marker = "extra == 'live'", specifier = ">=2.15.0" }, { name = "pygments", specifier = ">=2.20.0" }, @@ -1299,6 +1309,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/b2/b096ccce418882fbfda4f7496f9357aaa9a5af1896a9a7f60d9f2b275a06/grpcio-1.78.0-cp314-cp314-win_amd64.whl", hash = "sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb", size = 4929852, upload-time = "2026-02-06T09:56:45.885Z" }, ] +[[package]] +name = "grpcio-health-checking" +version = "1.78.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/ac/8eb871f4e47b11abfe45497e6187a582ec680ccd7232706d228474a8c7a5/grpcio_health_checking-1.78.0.tar.gz", hash = "sha256:78526d5c60b9b99fd18954b89f86d70033c702e96ad6ccc9749baf16136979b3", size = 17008, upload-time = "2026-02-06T10:01:47.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/30/dbaf47e2210697e2923b49eb62a6a2c07d5ee55bb40cff1e6cc0c5bb22e1/grpcio_health_checking-1.78.0-py3-none-any.whl", hash = "sha256:309798c098c5de72a9bff7172d788fdf309d246d231db9955b32e7c1c773fbeb", size = 19010, upload-time = "2026-02-06T10:01:37.949Z" }, +] + +[[package]] +name = "grpcio-reflection" +version = "1.78.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/06/337546aae558675f79cae2a8c1ce0c9b1952cbc5c28b01878f68d040f5bb/grpcio_reflection-1.78.0.tar.gz", hash = "sha256:e6e60c0b85dbcdf963b4d4d150c0f1d238ba891d805b575c52c0365d07fc0c40", size = 19098, upload-time = "2026-02-06T10:01:52.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/6d/4d095d27ccd049865ecdafc467754e9e47ad0f677a30dda969c3590f6582/grpcio_reflection-1.78.0-py3-none-any.whl", hash = "sha256:06fcfde9e6888cdd12e9dd1cf6dc7c440c2e9acf420f696ccbe008672ed05b60", size = 22800, upload-time = "2026-02-06T10:01:33.822Z" }, +] + [[package]] name = "grpcio-status" version = "1.78.0" @@ -2288,6 +2324,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/a1/9314e621c143e4d82a5bf7a43c2ff7a745d31023506336857607c8c543cc/opentelemetry_instrumentation-0.63b1-py3-none-any.whl", hash = "sha256:f1986716d52cc316ea5f60189098726a9071d8ecc0eee96c9ed110be08bade9c", size = 35577, upload-time = "2026-05-21T16:34:56.818Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-grpc" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/84/7c297486a6e9b99db041e5bd47f054aab29422be69659272908e292ed57a/opentelemetry_instrumentation_grpc-0.63b1.tar.gz", hash = "sha256:48d93e06872208a15ed720555ded9f366039924f41f3e16ea0da6ec937701e84", size = 31746, upload-time = "2026-05-21T16:36:33.561Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/2a/f4ad01d283550affaf3fa434671e3f6625095a4de90b9156bebaa4426aba/opentelemetry_instrumentation_grpc-0.63b1-py3-none-any.whl", hash = "sha256:40d30a2c5e97f90a28a8c4e688e368b4dcacc7654d358489cea92744311ff8d9", size = 24244, upload-time = "2026-05-21T16:35:31.098Z" }, +] + [[package]] name = "opentelemetry-instrumentation-openai-agents-v2" version = "0.1.0"